##// END OF EJS Templates
schain
RSS
Bug saving plots when throttle, add utcoffset to HDFReader
jespinoza -
r1363:1c8e307bd47c
parent child
Show More
Side by Side Unified
Context file:
r1363:1c8e307bd47c Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,691 +1,693
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """Base class to create plot operations
5 """Base class to create plot operations
6
6
7 """
7 """
8
8
9 import os
9 import os
10 import sys
10 import sys
11 import zmq
11 import zmq
12 import time
12 import time
13 import numpy
13 import numpy
14 import datetime
14 import datetime
15 from collections import deque
15 from collections import deque
16 from functools import wraps
16 from functools import wraps
17 from threading import Thread
17 from threading import Thread
18 import matplotlib
18 import matplotlib
19
19
20 if 'BACKEND' in os.environ:
20 if 'BACKEND' in os.environ:
21 matplotlib.use(os.environ['BACKEND'])
21 matplotlib.use(os.environ['BACKEND'])
22 elif 'linux' in sys.platform:
22 elif 'linux' in sys.platform:
23 matplotlib.use("TkAgg")
23 matplotlib.use("TkAgg")
24 elif 'darwin' in sys.platform:
24 elif 'darwin' in sys.platform:
25 matplotlib.use('MacOSX')
25 matplotlib.use('MacOSX')
26 else:
26 else:
27 from schainpy.utils import log
27 from schainpy.utils import log
28 log.warning('Using default Backend="Agg"', 'INFO')
28 log.warning('Using default Backend="Agg"', 'INFO')
29 matplotlib.use('Agg')
29 matplotlib.use('Agg')
30
30
31 import matplotlib.pyplot as plt
31 import matplotlib.pyplot as plt
32 from matplotlib.patches import Polygon
32 from matplotlib.patches import Polygon
33 from mpl_toolkits.axes_grid1 import make_axes_locatable
33 from mpl_toolkits.axes_grid1 import make_axes_locatable
34 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
34 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
35
35
36 from schainpy.model.data.jrodata import PlotterData
36 from schainpy.model.data.jrodata import PlotterData
37 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
37 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
38 from schainpy.utils import log
38 from schainpy.utils import log
39
39
40 jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
40 jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
41 blu_values = matplotlib.pyplot.get_cmap(
41 blu_values = matplotlib.pyplot.get_cmap(
42 'seismic_r', 20)(numpy.arange(20))[10:15]
42 'seismic_r', 20)(numpy.arange(20))[10:15]
43 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
43 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
44 'jro', numpy.vstack((blu_values, jet_values)))
44 'jro', numpy.vstack((blu_values, jet_values)))
45 matplotlib.pyplot.register_cmap(cmap=ncmap)
45 matplotlib.pyplot.register_cmap(cmap=ncmap)
46
46
47 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis',
47 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis',
48 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
48 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
49
49
50 EARTH_RADIUS = 6.3710e3
50 EARTH_RADIUS = 6.3710e3
51
51
52 def ll2xy(lat1, lon1, lat2, lon2):
52 def ll2xy(lat1, lon1, lat2, lon2):
53
53
54 p = 0.017453292519943295
54 p = 0.017453292519943295
55 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
55 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
56 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
56 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
57 r = 12742 * numpy.arcsin(numpy.sqrt(a))
57 r = 12742 * numpy.arcsin(numpy.sqrt(a))
58 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
58 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
59 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
59 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
60 theta = -theta + numpy.pi/2
60 theta = -theta + numpy.pi/2
61 return r*numpy.cos(theta), r*numpy.sin(theta)
61 return r*numpy.cos(theta), r*numpy.sin(theta)
62
62
63
63
64 def km2deg(km):
64 def km2deg(km):
65 '''
65 '''
66 Convert distance in km to degrees
66 Convert distance in km to degrees
67 '''
67 '''
68
68
69 return numpy.rad2deg(km/EARTH_RADIUS)
69 return numpy.rad2deg(km/EARTH_RADIUS)
70
70
71
71
72 def figpause(interval):
72 def figpause(interval):
73 backend = plt.rcParams['backend']
73 backend = plt.rcParams['backend']
74 if backend in matplotlib.rcsetup.interactive_bk:
74 if backend in matplotlib.rcsetup.interactive_bk:
75 figManager = matplotlib._pylab_helpers.Gcf.get_active()
75 figManager = matplotlib._pylab_helpers.Gcf.get_active()
76 if figManager is not None:
76 if figManager is not None:
77 canvas = figManager.canvas
77 canvas = figManager.canvas
78 if canvas.figure.stale:
78 if canvas.figure.stale:
79 canvas.draw()
79 canvas.draw()
80 try:
80 try:
81 canvas.start_event_loop(interval)
81 canvas.start_event_loop(interval)
82 except:
82 except:
83 pass
83 pass
84 return
84 return
85
85
86 def popup(message):
86 def popup(message):
87 '''
87 '''
88 '''
88 '''
89
89
90 fig = plt.figure(figsize=(12, 8), facecolor='r')
90 fig = plt.figure(figsize=(12, 8), facecolor='r')
91 text = '\n'.join([s.strip() for s in message.split(':')])
91 text = '\n'.join([s.strip() for s in message.split(':')])
92 fig.text(0.01, 0.5, text, ha='left', va='center',
92 fig.text(0.01, 0.5, text, ha='left', va='center',
93 size='20', weight='heavy', color='w')
93 size='20', weight='heavy', color='w')
94 fig.show()
94 fig.show()
95 figpause(1000)
95 figpause(1000)
96
96
97
97
98 class Throttle(object):
98 class Throttle(object):
99 '''
99 '''
100 Decorator that prevents a function from being called more than once every
100 Decorator that prevents a function from being called more than once every
101 time period.
101 time period.
102 To create a function that cannot be called more than once a minute, but
102 To create a function that cannot be called more than once a minute, but
103 will sleep until it can be called:
103 will sleep until it can be called:
104 @Throttle(minutes=1)
104 @Throttle(minutes=1)
105 def foo():
105 def foo():
106 pass
106 pass
107
107
108 for i in range(10):
108 for i in range(10):
109 foo()
109 foo()
110 print "This function has run %s times." % i
110 print "This function has run %s times." % i
111 '''
111 '''
112
112
113 def __init__(self, seconds=0, minutes=0, hours=0):
113 def __init__(self, seconds=0, minutes=0, hours=0):
114 self.throttle_period = datetime.timedelta(
114 self.throttle_period = datetime.timedelta(
115 seconds=seconds, minutes=minutes, hours=hours
115 seconds=seconds, minutes=minutes, hours=hours
116 )
116 )
117
117
118 self.time_of_last_call = datetime.datetime.min
118 self.time_of_last_call = datetime.datetime.min
119
119
120 def __call__(self, fn):
120 def __call__(self, fn):
121 @wraps(fn)
121 @wraps(fn)
122 def wrapper(*args, **kwargs):
122 def wrapper(*args, **kwargs):
123 coerce = kwargs.pop('coerce', None)
123 coerce = kwargs.pop('coerce', None)
124 if coerce:
124 if coerce:
125 self.time_of_last_call = datetime.datetime.now()
125 self.time_of_last_call = datetime.datetime.now()
126 return fn(*args, **kwargs)
126 return fn(*args, **kwargs)
127 else:
127 else:
128 now = datetime.datetime.now()
128 now = datetime.datetime.now()
129 time_since_last_call = now - self.time_of_last_call
129 time_since_last_call = now - self.time_of_last_call
130 time_left = self.throttle_period - time_since_last_call
130 time_left = self.throttle_period - time_since_last_call
131
131
132 if time_left > datetime.timedelta(seconds=0):
132 if time_left > datetime.timedelta(seconds=0):
133 return
133 return
134
134
135 self.time_of_last_call = datetime.datetime.now()
135 self.time_of_last_call = datetime.datetime.now()
136 return fn(*args, **kwargs)
136 return fn(*args, **kwargs)
137
137
138 return wrapper
138 return wrapper
139
139
140 def apply_throttle(value):
140 def apply_throttle(value):
141
141
142 @Throttle(seconds=value)
142 @Throttle(seconds=value)
143 def fnThrottled(fn):
143 def fnThrottled(fn):
144 fn()
144 fn()
145
145
146 return fnThrottled
146 return fnThrottled
147
147
148
148
149 @MPDecorator
149 @MPDecorator
150 class Plot(Operation):
150 class Plot(Operation):
151 """Base class for Schain plotting operations
151 """Base class for Schain plotting operations
152
152
153 This class should never be use directtly you must subclass a new operation,
153 This class should never be use directtly you must subclass a new operation,
154 children classes must be defined as follow:
154 children classes must be defined as follow:
155
155
156 ExamplePlot(Plot):
156 ExamplePlot(Plot):
157
157
158 CODE = 'code'
158 CODE = 'code'
159 colormap = 'jet'
159 colormap = 'jet'
160 plot_type = 'pcolor' # options are ('pcolor', 'pcolorbuffer', 'scatter', 'scatterbuffer')
160 plot_type = 'pcolor' # options are ('pcolor', 'pcolorbuffer', 'scatter', 'scatterbuffer')
161
161
162 def setup(self):
162 def setup(self):
163 pass
163 pass
164
164
165 def plot(self):
165 def plot(self):
166 pass
166 pass
167
167
168 """
168 """
169
169
170 CODE = 'Figure'
170 CODE = 'Figure'
171 colormap = 'jet'
171 colormap = 'jet'
172 bgcolor = 'white'
172 bgcolor = 'white'
173 buffering = True
173 buffering = True
174 __missing = 1E30
174 __missing = 1E30
175
175
176 __attrs__ = ['show', 'save', 'ymin', 'ymax', 'zmin', 'zmax', 'title',
176 __attrs__ = ['show', 'save', 'ymin', 'ymax', 'zmin', 'zmax', 'title',
177 'showprofile']
177 'showprofile']
178
178
179 def __init__(self):
179 def __init__(self):
180
180
181 Operation.__init__(self)
181 Operation.__init__(self)
182 self.isConfig = False
182 self.isConfig = False
183 self.isPlotConfig = False
183 self.isPlotConfig = False
184 self.save_time = 0
184 self.save_time = 0
185 self.sender_time = 0
185 self.sender_time = 0
186 self.data = None
186 self.data = None
187 self.firsttime = True
187 self.firsttime = True
188 self.sender_queue = deque(maxlen=10)
188 self.sender_queue = deque(maxlen=10)
189 self.plots_adjust = {'left': 0.125, 'right': 0.9, 'bottom': 0.15, 'top': 0.9, 'wspace': 0.2, 'hspace': 0.2}
189 self.plots_adjust = {'left': 0.125, 'right': 0.9, 'bottom': 0.15, 'top': 0.9, 'wspace': 0.2, 'hspace': 0.2}
190
190
191 def __fmtTime(self, x, pos):
191 def __fmtTime(self, x, pos):
192 '''
192 '''
193 '''
193 '''
194
194
195 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
195 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
196
196
197 def __setup(self, **kwargs):
197 def __setup(self, **kwargs):
198 '''
198 '''
199 Initialize variables
199 Initialize variables
200 '''
200 '''
201
201
202 self.figures = []
202 self.figures = []
203 self.axes = []
203 self.axes = []
204 self.cb_axes = []
204 self.cb_axes = []
205 self.localtime = kwargs.pop('localtime', True)
205 self.localtime = kwargs.pop('localtime', True)
206 self.show = kwargs.get('show', True)
206 self.show = kwargs.get('show', True)
207 self.save = kwargs.get('save', False)
207 self.save = kwargs.get('save', False)
208 self.save_period = kwargs.get('save_period', 0)
208 self.save_period = kwargs.get('save_period', 0)
209 self.colormap = kwargs.get('colormap', self.colormap)
209 self.colormap = kwargs.get('colormap', self.colormap)
210 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
210 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
211 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
211 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
212 self.colormaps = kwargs.get('colormaps', None)
212 self.colormaps = kwargs.get('colormaps', None)
213 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
213 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
214 self.showprofile = kwargs.get('showprofile', False)
214 self.showprofile = kwargs.get('showprofile', False)
215 self.title = kwargs.get('wintitle', self.CODE.upper())
215 self.title = kwargs.get('wintitle', self.CODE.upper())
216 self.cb_label = kwargs.get('cb_label', None)
216 self.cb_label = kwargs.get('cb_label', None)
217 self.cb_labels = kwargs.get('cb_labels', None)
217 self.cb_labels = kwargs.get('cb_labels', None)
218 self.labels = kwargs.get('labels', None)
218 self.labels = kwargs.get('labels', None)
219 self.xaxis = kwargs.get('xaxis', 'frequency')
219 self.xaxis = kwargs.get('xaxis', 'frequency')
220 self.zmin = kwargs.get('zmin', None)
220 self.zmin = kwargs.get('zmin', None)
221 self.zmax = kwargs.get('zmax', None)
221 self.zmax = kwargs.get('zmax', None)
222 self.zlimits = kwargs.get('zlimits', None)
222 self.zlimits = kwargs.get('zlimits', None)
223 self.xmin = kwargs.get('xmin', None)
223 self.xmin = kwargs.get('xmin', None)
224 self.xmax = kwargs.get('xmax', None)
224 self.xmax = kwargs.get('xmax', None)
225 self.xrange = kwargs.get('xrange', 12)
225 self.xrange = kwargs.get('xrange', 12)
226 self.xscale = kwargs.get('xscale', None)
226 self.xscale = kwargs.get('xscale', None)
227 self.ymin = kwargs.get('ymin', None)
227 self.ymin = kwargs.get('ymin', None)
228 self.ymax = kwargs.get('ymax', None)
228 self.ymax = kwargs.get('ymax', None)
229 self.yscale = kwargs.get('yscale', None)
229 self.yscale = kwargs.get('yscale', None)
230 self.xlabel = kwargs.get('xlabel', None)
230 self.xlabel = kwargs.get('xlabel', None)
231 self.attr_time = kwargs.get('attr_time', 'utctime')
231 self.attr_time = kwargs.get('attr_time', 'utctime')
232 self.attr_data = kwargs.get('attr_data', 'data_param')
232 self.attr_data = kwargs.get('attr_data', 'data_param')
233 self.decimation = kwargs.get('decimation', None)
233 self.decimation = kwargs.get('decimation', None)
234 self.oneFigure = kwargs.get('oneFigure', True)
234 self.oneFigure = kwargs.get('oneFigure', True)
235 self.width = kwargs.get('width', None)
235 self.width = kwargs.get('width', None)
236 self.height = kwargs.get('height', None)
236 self.height = kwargs.get('height', None)
237 self.colorbar = kwargs.get('colorbar', True)
237 self.colorbar = kwargs.get('colorbar', True)
238 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
238 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
239 self.channels = kwargs.get('channels', None)
239 self.channels = kwargs.get('channels', None)
240 self.titles = kwargs.get('titles', [])
240 self.titles = kwargs.get('titles', [])
241 self.polar = False
241 self.polar = False
242 self.type = kwargs.get('type', 'iq')
242 self.type = kwargs.get('type', 'iq')
243 self.grid = kwargs.get('grid', False)
243 self.grid = kwargs.get('grid', False)
244 self.pause = kwargs.get('pause', False)
244 self.pause = kwargs.get('pause', False)
245 self.save_code = kwargs.get('save_code', self.CODE)
245 self.save_code = kwargs.get('save_code', self.CODE)
246 self.throttle = kwargs.get('throttle', 0)
246 self.throttle = kwargs.get('throttle', 0)
247 self.exp_code = kwargs.get('exp_code', None)
247 self.exp_code = kwargs.get('exp_code', None)
248 self.server = kwargs.get('server', False)
248 self.server = kwargs.get('server', False)
249 self.sender_period = kwargs.get('sender_period', 60)
249 self.sender_period = kwargs.get('sender_period', 60)
250 self.tag = kwargs.get('tag', '')
250 self.tag = kwargs.get('tag', '')
251 self.height_index = kwargs.get('height_index', None)
251 self.height_index = kwargs.get('height_index', None)
252 self.__throttle_plot = apply_throttle(self.throttle)
252 self.__throttle_plot = apply_throttle(self.throttle)
253 code = self.attr_data if self.attr_data else self.CODE
253 code = self.attr_data if self.attr_data else self.CODE
254 self.data = PlotterData(self.CODE, self.exp_code, self.localtime)
254 self.data = PlotterData(self.CODE, self.exp_code, self.localtime)
255
255
256 if self.server:
256 if self.server:
257 if not self.server.startswith('tcp://'):
257 if not self.server.startswith('tcp://'):
258 self.server = 'tcp://{}'.format(self.server)
258 self.server = 'tcp://{}'.format(self.server)
259 log.success(
259 log.success(
260 'Sending to server: {}'.format(self.server),
260 'Sending to server: {}'.format(self.server),
261 self.name
261 self.name
262 )
262 )
263
263
264 if isinstance(self.attr_data, str):
264 if isinstance(self.attr_data, str):
265 self.attr_data = [self.attr_data]
265 self.attr_data = [self.attr_data]
266
266
267 def __setup_plot(self):
267 def __setup_plot(self):
268 '''
268 '''
269 Common setup for all figures, here figures and axes are created
269 Common setup for all figures, here figures and axes are created
270 '''
270 '''
271
271
272 self.setup()
272 self.setup()
273
273
274 self.time_label = 'LT' if self.localtime else 'UTC'
274 self.time_label = 'LT' if self.localtime else 'UTC'
275
275
276 if self.width is None:
276 if self.width is None:
277 self.width = 8
277 self.width = 8
278
278
279 self.figures = []
279 self.figures = []
280 self.axes = []
280 self.axes = []
281 self.cb_axes = []
281 self.cb_axes = []
282 self.pf_axes = []
282 self.pf_axes = []
283 self.cmaps = []
283 self.cmaps = []
284
284
285 size = '15%' if self.ncols == 1 else '30%'
285 size = '15%' if self.ncols == 1 else '30%'
286 pad = '4%' if self.ncols == 1 else '8%'
286 pad = '4%' if self.ncols == 1 else '8%'
287
287
288 if self.oneFigure:
288 if self.oneFigure:
289 if self.height is None:
289 if self.height is None:
290 self.height = 1.4 * self.nrows + 1
290 self.height = 1.4 * self.nrows + 1
291 fig = plt.figure(figsize=(self.width, self.height),
291 fig = plt.figure(figsize=(self.width, self.height),
292 edgecolor='k',
292 edgecolor='k',
293 facecolor='w')
293 facecolor='w')
294 self.figures.append(fig)
294 self.figures.append(fig)
295 for n in range(self.nplots):
295 for n in range(self.nplots):
296 ax = fig.add_subplot(self.nrows, self.ncols,
296 ax = fig.add_subplot(self.nrows, self.ncols,
297 n + 1, polar=self.polar)
297 n + 1, polar=self.polar)
298 ax.tick_params(labelsize=8)
298 ax.tick_params(labelsize=8)
299 ax.firsttime = True
299 ax.firsttime = True
300 ax.index = 0
300 ax.index = 0
301 ax.press = None
301 ax.press = None
302 self.axes.append(ax)
302 self.axes.append(ax)
303 if self.showprofile:
303 if self.showprofile:
304 cax = self.__add_axes(ax, size=size, pad=pad)
304 cax = self.__add_axes(ax, size=size, pad=pad)
305 cax.tick_params(labelsize=8)
305 cax.tick_params(labelsize=8)
306 self.pf_axes.append(cax)
306 self.pf_axes.append(cax)
307 else:
307 else:
308 if self.height is None:
308 if self.height is None:
309 self.height = 3
309 self.height = 3
310 for n in range(self.nplots):
310 for n in range(self.nplots):
311 fig = plt.figure(figsize=(self.width, self.height),
311 fig = plt.figure(figsize=(self.width, self.height),
312 edgecolor='k',
312 edgecolor='k',
313 facecolor='w')
313 facecolor='w')
314 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
314 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
315 ax.tick_params(labelsize=8)
315 ax.tick_params(labelsize=8)
316 ax.firsttime = True
316 ax.firsttime = True
317 ax.index = 0
317 ax.index = 0
318 ax.press = None
318 ax.press = None
319 self.figures.append(fig)
319 self.figures.append(fig)
320 self.axes.append(ax)
320 self.axes.append(ax)
321 if self.showprofile:
321 if self.showprofile:
322 cax = self.__add_axes(ax, size=size, pad=pad)
322 cax = self.__add_axes(ax, size=size, pad=pad)
323 cax.tick_params(labelsize=8)
323 cax.tick_params(labelsize=8)
324 self.pf_axes.append(cax)
324 self.pf_axes.append(cax)
325
325
326 for n in range(self.nrows):
326 for n in range(self.nrows):
327 print(self.nrows)
328 if self.colormaps is not None:
327 if self.colormaps is not None:
329 cmap = plt.get_cmap(self.colormaps[n])
328 cmap = plt.get_cmap(self.colormaps[n])
330 else:
329 else:
331 cmap = plt.get_cmap(self.colormap)
330 cmap = plt.get_cmap(self.colormap)
332 cmap.set_bad(self.bgcolor, 1.)
331 cmap.set_bad(self.bgcolor, 1.)
333 self.cmaps.append(cmap)
332 self.cmaps.append(cmap)
334
333
335 def __add_axes(self, ax, size='30%', pad='8%'):
334 def __add_axes(self, ax, size='30%', pad='8%'):
336 '''
335 '''
337 Add new axes to the given figure
336 Add new axes to the given figure
338 '''
337 '''
339 divider = make_axes_locatable(ax)
338 divider = make_axes_locatable(ax)
340 nax = divider.new_horizontal(size=size, pad=pad)
339 nax = divider.new_horizontal(size=size, pad=pad)
341 ax.figure.add_axes(nax)
340 ax.figure.add_axes(nax)
342 return nax
341 return nax
343
342
344 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
343 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
345 '''
344 '''
346 Create a masked array for missing data
345 Create a masked array for missing data
347 '''
346 '''
348 if x_buffer.shape[0] < 2:
347 if x_buffer.shape[0] < 2:
349 return x_buffer, y_buffer, z_buffer
348 return x_buffer, y_buffer, z_buffer
350
349
351 deltas = x_buffer[1:] - x_buffer[0:-1]
350 deltas = x_buffer[1:] - x_buffer[0:-1]
352 x_median = numpy.median(deltas)
351 x_median = numpy.median(deltas)
353
352
354 index = numpy.where(deltas > 5 * x_median)
353 index = numpy.where(deltas > 5 * x_median)
355
354
356 if len(index[0]) != 0:
355 if len(index[0]) != 0:
357 z_buffer[::, index[0], ::] = self.__missing
356 z_buffer[::, index[0], ::] = self.__missing
358 z_buffer = numpy.ma.masked_inside(z_buffer,
357 z_buffer = numpy.ma.masked_inside(z_buffer,
359 0.99 * self.__missing,
358 0.99 * self.__missing,
360 1.01 * self.__missing)
359 1.01 * self.__missing)
361
360
362 return x_buffer, y_buffer, z_buffer
361 return x_buffer, y_buffer, z_buffer
363
362
364 def decimate(self):
363 def decimate(self):
365
364
366 # dx = int(len(self.x)/self.__MAXNUMX) + 1
365 # dx = int(len(self.x)/self.__MAXNUMX) + 1
367 dy = int(len(self.y) / self.decimation) + 1
366 dy = int(len(self.y) / self.decimation) + 1
368
367
369 # x = self.x[::dx]
368 # x = self.x[::dx]
370 x = self.x
369 x = self.x
371 y = self.y[::dy]
370 y = self.y[::dy]
372 z = self.z[::, ::, ::dy]
371 z = self.z[::, ::, ::dy]
373
372
374 return x, y, z
373 return x, y, z
375
374
376 def format(self):
375 def format(self):
377 '''
376 '''
378 Set min and max values, labels, ticks and titles
377 Set min and max values, labels, ticks and titles
379 '''
378 '''
380
379
381 for n, ax in enumerate(self.axes):
380 for n, ax in enumerate(self.axes):
382 if ax.firsttime:
381 if ax.firsttime:
383 if self.xaxis != 'time':
382 if self.xaxis != 'time':
384 xmin = self.xmin
383 xmin = self.xmin
385 xmax = self.xmax
384 xmax = self.xmax
386 else:
385 else:
387 xmin = self.tmin
386 xmin = self.tmin
388 xmax = self.tmin + self.xrange*60*60
387 xmax = self.tmin + self.xrange*60*60
389 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
388 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
390 ax.xaxis.set_major_locator(LinearLocator(9))
389 ax.xaxis.set_major_locator(LinearLocator(9))
391 ymin = self.ymin if self.ymin is not None else numpy.nanmin(self.y[numpy.isfinite(self.y)])
390 ymin = self.ymin if self.ymin is not None else numpy.nanmin(self.y[numpy.isfinite(self.y)])
392 ymax = self.ymax if self.ymax is not None else numpy.nanmax(self.y[numpy.isfinite(self.y)])
391 ymax = self.ymax if self.ymax is not None else numpy.nanmax(self.y[numpy.isfinite(self.y)])
393 ax.set_facecolor(self.bgcolor)
392 ax.set_facecolor(self.bgcolor)
394 if self.xscale:
393 if self.xscale:
395 ax.xaxis.set_major_formatter(FuncFormatter(
394 ax.xaxis.set_major_formatter(FuncFormatter(
396 lambda x, pos: '{0:g}'.format(x*self.xscale)))
395 lambda x, pos: '{0:g}'.format(x*self.xscale)))
397 if self.yscale:
396 if self.yscale:
398 ax.yaxis.set_major_formatter(FuncFormatter(
397 ax.yaxis.set_major_formatter(FuncFormatter(
399 lambda x, pos: '{0:g}'.format(x*self.yscale)))
398 lambda x, pos: '{0:g}'.format(x*self.yscale)))
400 if self.xlabel is not None:
399 if self.xlabel is not None:
401 ax.set_xlabel(self.xlabel)
400 ax.set_xlabel(self.xlabel)
402 if self.ylabel is not None:
401 if self.ylabel is not None:
403 ax.set_ylabel(self.ylabel)
402 ax.set_ylabel(self.ylabel)
404 if self.showprofile:
403 if self.showprofile:
405 self.pf_axes[n].set_ylim(ymin, ymax)
404 self.pf_axes[n].set_ylim(ymin, ymax)
406 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
405 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
407 self.pf_axes[n].set_xlabel('dB')
406 self.pf_axes[n].set_xlabel('dB')
408 self.pf_axes[n].grid(b=True, axis='x')
407 self.pf_axes[n].grid(b=True, axis='x')
409 [tick.set_visible(False)
408 [tick.set_visible(False)
410 for tick in self.pf_axes[n].get_yticklabels()]
409 for tick in self.pf_axes[n].get_yticklabels()]
411 if self.colorbar:
410 if self.colorbar:
412 ax.cbar = plt.colorbar(
411 ax.cbar = plt.colorbar(
413 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
412 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
414 ax.cbar.ax.tick_params(labelsize=8)
413 ax.cbar.ax.tick_params(labelsize=8)
415 ax.cbar.ax.press = None
414 ax.cbar.ax.press = None
416 if self.cb_label:
415 if self.cb_label:
417 ax.cbar.set_label(self.cb_label, size=8)
416 ax.cbar.set_label(self.cb_label, size=8)
418 elif self.cb_labels:
417 elif self.cb_labels:
419 ax.cbar.set_label(self.cb_labels[n], size=8)
418 ax.cbar.set_label(self.cb_labels[n], size=8)
420 else:
419 else:
421 ax.cbar = None
420 ax.cbar = None
422 ax.set_xlim(xmin, xmax)
421 ax.set_xlim(xmin, xmax)
423 ax.set_ylim(ymin, ymax)
422 ax.set_ylim(ymin, ymax)
424 ax.firsttime = False
423 ax.firsttime = False
425 if self.grid:
424 if self.grid:
426 ax.grid(True)
425 ax.grid(True)
427 if not self.polar:
426 if not self.polar:
428 ax.set_title('{} {} {}'.format(
427 ax.set_title('{} {} {}'.format(
429 self.titles[n],
428 self.titles[n],
430 self.getDateTime(self.data.max_time).strftime(
429 self.getDateTime(self.data.max_time).strftime(
431 '%Y-%m-%d %H:%M:%S'),
430 '%Y-%m-%d %H:%M:%S'),
432 self.time_label),
431 self.time_label),
433 size=8)
432 size=8)
434 else:
433 else:
435 ax.set_title('{}'.format(self.titles[n]), size=8)
434 ax.set_title('{}'.format(self.titles[n]), size=8)
436 ax.set_ylim(0, 90)
435 ax.set_ylim(0, 90)
437 ax.set_yticks(numpy.arange(0, 90, 20))
436 ax.set_yticks(numpy.arange(0, 90, 20))
438 ax.yaxis.labelpad = 40
437 ax.yaxis.labelpad = 40
439
438
440 if self.firsttime:
439 if self.firsttime:
441 for n, fig in enumerate(self.figures):
440 for n, fig in enumerate(self.figures):
442 fig.subplots_adjust(**self.plots_adjust)
441 fig.subplots_adjust(**self.plots_adjust)
443 self.firsttime = False
442 self.firsttime = False
444
443
445 def clear_figures(self):
444 def clear_figures(self):
446 '''
445 '''
447 Reset axes for redraw plots
446 Reset axes for redraw plots
448 '''
447 '''
449
448
450 for ax in self.axes+self.pf_axes+self.cb_axes:
449 for ax in self.axes+self.pf_axes+self.cb_axes:
451 ax.clear()
450 ax.clear()
452 ax.firsttime = True
451 ax.firsttime = True
453 if hasattr(ax, 'cbar') and ax.cbar:
452 if hasattr(ax, 'cbar') and ax.cbar:
454 ax.cbar.remove()
453 ax.cbar.remove()
455
454
456 def __plot(self):
455 def __plot(self):
457 '''
456 '''
458 Main function to plot, format and save figures
457 Main function to plot, format and save figures
459 '''
458 '''
460
459
461 self.plot()
460 self.plot()
462 self.format()
461 self.format()
463
462
464 for n, fig in enumerate(self.figures):
463 for n, fig in enumerate(self.figures):
465 if self.nrows == 0 or self.nplots == 0:
464 if self.nrows == 0 or self.nplots == 0:
466 log.warning('No data', self.name)
465 log.warning('No data', self.name)
467 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
466 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
468 fig.canvas.manager.set_window_title(self.CODE)
467 fig.canvas.manager.set_window_title(self.CODE)
469 continue
468 continue
470
469
471 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
470 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
472 self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
471 self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
473 fig.canvas.draw()
472 fig.canvas.draw()
474 if self.show:
473 if self.show:
475 fig.show()
474 fig.show()
476 figpause(0.01)
475 figpause(0.01)
477
476
478 if self.save:
477 if self.save:
479 self.save_figure(n)
478 self.save_figure(n)
480
479
481 if self.server:
480 if self.server:
482 self.send_to_server()
481 self.send_to_server()
483
482
484 def __update(self, dataOut, timestamp):
483 def __update(self, dataOut, timestamp):
485 '''
484 '''
486 '''
485 '''
487
486
488 metadata = {
487 metadata = {
489 'yrange': dataOut.heightList,
488 'yrange': dataOut.heightList,
490 'interval': dataOut.timeInterval,
489 'interval': dataOut.timeInterval,
491 'channels': dataOut.channelList
490 'channels': dataOut.channelList
492 }
491 }
493
492
494 data, meta = self.update(dataOut)
493 data, meta = self.update(dataOut)
495 metadata.update(meta)
494 metadata.update(meta)
496 self.data.update(data, timestamp, metadata)
495 self.data.update(data, timestamp, metadata)
497
496
498 def save_figure(self, n):
497 def save_figure(self, n):
499 '''
498 '''
500 '''
499 '''
501
500
502 if (self.data.max_time - self.save_time) <= self.save_period:
501 if (self.data.max_time - self.save_time) <= self.save_period:
503 return
502 return
504
503
505 self.save_time = self.data.max_time
504 self.save_time = self.data.max_time
506
505
507 fig = self.figures[n]
506 fig = self.figures[n]
508
507
509 if self.throttle == 0:
508 if self.throttle == 0:
510 figname = os.path.join(
509 figname = os.path.join(
511 self.save,
510 self.save,
512 self.save_code,
511 self.save_code,
513 '{}_{}.png'.format(
512 '{}_{}.png'.format(
514 self.save_code,
513 self.save_code,
515 self.getDateTime(self.data.max_time).strftime(
514 self.getDateTime(self.data.max_time).strftime(
516 '%Y%m%d_%H%M%S'
515 '%Y%m%d_%H%M%S'
517 ),
516 ),
518 )
517 )
519 )
518 )
520 log.log('Saving figure: {}'.format(figname), self.name)
519 log.log('Saving figure: {}'.format(figname), self.name)
521 if not os.path.isdir(os.path.dirname(figname)):
520 if not os.path.isdir(os.path.dirname(figname)):
522 os.makedirs(os.path.dirname(figname))
521 os.makedirs(os.path.dirname(figname))
523 fig.savefig(figname)
522 fig.savefig(figname)
524
523
525 figname = os.path.join(
524 figname = os.path.join(
526 self.save,
525 self.save,
527 '{}_{}.png'.format(
526 '{}_{}.png'.format(
528 self.save_code,
527 self.save_code,
529 self.getDateTime(self.data.min_time).strftime(
528 self.getDateTime(self.data.min_time).strftime(
530 '%Y%m%d'
529 '%Y%m%d'
531 ),
530 ),
532 )
531 )
533 )
532 )
533 log.log('Saving figure: {}'.format(figname), self.name)
534 if not os.path.isdir(os.path.dirname(figname)):
535 os.makedirs(os.path.dirname(figname))
534 fig.savefig(figname)
536 fig.savefig(figname)
535
537
536 def send_to_server(self):
538 def send_to_server(self):
537 '''
539 '''
538 '''
540 '''
539
541
540 if self.exp_code == None:
542 if self.exp_code == None:
541 log.warning('Missing `exp_code` skipping sending to server...')
543 log.warning('Missing `exp_code` skipping sending to server...')
542
544
543 last_time = self.data.max_time
545 last_time = self.data.max_time
544 interval = last_time - self.sender_time
546 interval = last_time - self.sender_time
545 if interval < self.sender_period:
547 if interval < self.sender_period:
546 return
548 return
547
549
548 self.sender_time = last_time
550 self.sender_time = last_time
549
551
550 attrs = ['titles', 'zmin', 'zmax', 'tag', 'ymin', 'ymax']
552 attrs = ['titles', 'zmin', 'zmax', 'tag', 'ymin', 'ymax']
551 for attr in attrs:
553 for attr in attrs:
552 value = getattr(self, attr)
554 value = getattr(self, attr)
553 if value:
555 if value:
554 if isinstance(value, (numpy.float32, numpy.float64)):
556 if isinstance(value, (numpy.float32, numpy.float64)):
555 value = round(float(value), 2)
557 value = round(float(value), 2)
556 self.data.meta[attr] = value
558 self.data.meta[attr] = value
557 if self.colormap == 'jet':
559 if self.colormap == 'jet':
558 self.data.meta['colormap'] = 'Jet'
560 self.data.meta['colormap'] = 'Jet'
559 elif 'RdBu' in self.colormap:
561 elif 'RdBu' in self.colormap:
560 self.data.meta['colormap'] = 'RdBu'
562 self.data.meta['colormap'] = 'RdBu'
561 else:
563 else:
562 self.data.meta['colormap'] = 'Viridis'
564 self.data.meta['colormap'] = 'Viridis'
563 self.data.meta['interval'] = int(interval)
565 self.data.meta['interval'] = int(interval)
564
566
565 self.sender_queue.append(last_time)
567 self.sender_queue.append(last_time)
566
568
567 while True:
569 while True:
568 try:
570 try:
569 tm = self.sender_queue.popleft()
571 tm = self.sender_queue.popleft()
570 except IndexError:
572 except IndexError:
571 break
573 break
572 msg = self.data.jsonify(tm, self.save_code, self.plot_type)
574 msg = self.data.jsonify(tm, self.save_code, self.plot_type)
573 self.socket.send_string(msg)
575 self.socket.send_string(msg)
574 socks = dict(self.poll.poll(2000))
576 socks = dict(self.poll.poll(2000))
575 if socks.get(self.socket) == zmq.POLLIN:
577 if socks.get(self.socket) == zmq.POLLIN:
576 reply = self.socket.recv_string()
578 reply = self.socket.recv_string()
577 if reply == 'ok':
579 if reply == 'ok':
578 log.log("Response from server ok", self.name)
580 log.log("Response from server ok", self.name)
579 time.sleep(0.1)
581 time.sleep(0.1)
580 continue
582 continue
581 else:
583 else:
582 log.warning(
584 log.warning(
583 "Malformed reply from server: {}".format(reply), self.name)
585 "Malformed reply from server: {}".format(reply), self.name)
584 else:
586 else:
585 log.warning(
587 log.warning(
586 "No response from server, retrying...", self.name)
588 "No response from server, retrying...", self.name)
587 self.sender_queue.appendleft(tm)
589 self.sender_queue.appendleft(tm)
588 self.socket.setsockopt(zmq.LINGER, 0)
590 self.socket.setsockopt(zmq.LINGER, 0)
589 self.socket.close()
591 self.socket.close()
590 self.poll.unregister(self.socket)
592 self.poll.unregister(self.socket)
591 self.socket = self.context.socket(zmq.REQ)
593 self.socket = self.context.socket(zmq.REQ)
592 self.socket.connect(self.server)
594 self.socket.connect(self.server)
593 self.poll.register(self.socket, zmq.POLLIN)
595 self.poll.register(self.socket, zmq.POLLIN)
594 break
596 break
595
597
596 def setup(self):
598 def setup(self):
597 '''
599 '''
598 This method should be implemented in the child class, the following
600 This method should be implemented in the child class, the following
599 attributes should be set:
601 attributes should be set:
600
602
601 self.nrows: number of rows
603 self.nrows: number of rows
602 self.ncols: number of cols
604 self.ncols: number of cols
603 self.nplots: number of plots (channels or pairs)
605 self.nplots: number of plots (channels or pairs)
604 self.ylabel: label for Y axes
606 self.ylabel: label for Y axes
605 self.titles: list of axes title
607 self.titles: list of axes title
606
608
607 '''
609 '''
608 raise NotImplementedError
610 raise NotImplementedError
609
611
610 def plot(self):
612 def plot(self):
611 '''
613 '''
612 Must be defined in the child class, the actual plotting method
614 Must be defined in the child class, the actual plotting method
613 '''
615 '''
614 raise NotImplementedError
616 raise NotImplementedError
615
617
616 def update(self, dataOut):
618 def update(self, dataOut):
617 '''
619 '''
618 Must be defined in the child class, update self.data with new data
620 Must be defined in the child class, update self.data with new data
619 '''
621 '''
620
622
621 data = {
623 data = {
622 self.CODE: getattr(dataOut, 'data_{}'.format(self.CODE))
624 self.CODE: getattr(dataOut, 'data_{}'.format(self.CODE))
623 }
625 }
624 meta = {}
626 meta = {}
625
627
626 return data, meta
628 return data, meta
627
629
628 def run(self, dataOut, **kwargs):
630 def run(self, dataOut, **kwargs):
629 '''
631 '''
630 Main plotting routine
632 Main plotting routine
631 '''
633 '''
632
634
633 if self.isConfig is False:
635 if self.isConfig is False:
634 self.__setup(**kwargs)
636 self.__setup(**kwargs)
635
637
636 if self.localtime:
638 if self.localtime:
637 self.getDateTime = datetime.datetime.fromtimestamp
639 self.getDateTime = datetime.datetime.fromtimestamp
638 else:
640 else:
639 self.getDateTime = datetime.datetime.utcfromtimestamp
641 self.getDateTime = datetime.datetime.utcfromtimestamp
640
642
641 self.data.setup()
643 self.data.setup()
642 self.isConfig = True
644 self.isConfig = True
643 if self.server:
645 if self.server:
644 self.context = zmq.Context()
646 self.context = zmq.Context()
645 self.socket = self.context.socket(zmq.REQ)
647 self.socket = self.context.socket(zmq.REQ)
646 self.socket.connect(self.server)
648 self.socket.connect(self.server)
647 self.poll = zmq.Poller()
649 self.poll = zmq.Poller()
648 self.poll.register(self.socket, zmq.POLLIN)
650 self.poll.register(self.socket, zmq.POLLIN)
649
651
650 tm = getattr(dataOut, self.attr_time)
652 tm = getattr(dataOut, self.attr_time)
651
653
652 if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60:
654 if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60:
653 self.save_time = tm
655 self.save_time = tm
654 self.__plot()
656 self.__plot()
655 self.tmin += self.xrange*60*60
657 self.tmin += self.xrange*60*60
656 self.data.setup()
658 self.data.setup()
657 self.clear_figures()
659 self.clear_figures()
658
660
659 self.__update(dataOut, tm)
661 self.__update(dataOut, tm)
660
662
661 if self.isPlotConfig is False:
663 if self.isPlotConfig is False:
662 self.__setup_plot()
664 self.__setup_plot()
663 self.isPlotConfig = True
665 self.isPlotConfig = True
664 if self.xaxis == 'time':
666 if self.xaxis == 'time':
665 dt = self.getDateTime(tm)
667 dt = self.getDateTime(tm)
666 if self.xmin is None:
668 if self.xmin is None:
667 self.tmin = tm
669 self.tmin = tm
668 self.xmin = dt.hour
670 self.xmin = dt.hour
669 minutes = (self.xmin-int(self.xmin)) * 60
671 minutes = (self.xmin-int(self.xmin)) * 60
670 seconds = (minutes - int(minutes)) * 60
672 seconds = (minutes - int(minutes)) * 60
671 self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) -
673 self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) -
672 datetime.datetime(1970, 1, 1)).total_seconds()
674 datetime.datetime(1970, 1, 1)).total_seconds()
673 if self.localtime:
675 if self.localtime:
674 self.tmin += time.timezone
676 self.tmin += time.timezone
675
677
676 if self.xmin is not None and self.xmax is not None:
678 if self.xmin is not None and self.xmax is not None:
677 self.xrange = self.xmax - self.xmin
679 self.xrange = self.xmax - self.xmin
678
680
679 if self.throttle == 0:
681 if self.throttle == 0:
680 self.__plot()
682 self.__plot()
681 else:
683 else:
682 self.__throttle_plot(self.__plot)#, coerce=coerce)
684 self.__throttle_plot(self.__plot)#, coerce=coerce)
683
685
684 def close(self):
686 def close(self):
685
687
686 if self.data and not self.data.flagNoData:
688 if self.data and not self.data.flagNoData:
687 self.save_time = self.data.max_time
689 self.save_time = 0
688 self.__plot()
690 self.__plot()
689 if self.data and not self.data.flagNoData and self.pause:
691 if self.data and not self.data.flagNoData and self.pause:
690 figpause(10)
692 figpause(10)
691
693
Show file before | Show file after | Hide comments
@@ -1,627 +1,626
1 import os
1 import os
2 import time
2 import time
3 import datetime
3 import datetime
4
4
5 import numpy
5 import numpy
6 import h5py
6 import h5py
7
7
8 import schainpy.admin
8 import schainpy.admin
9 from schainpy.model.data.jrodata import *
9 from schainpy.model.data.jrodata import *
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 from schainpy.utils import log
12 from schainpy.utils import log
13
13
14
14
15 class HDFReader(Reader, ProcessingUnit):
15 class HDFReader(Reader, ProcessingUnit):
16 """Processing unit to read HDF5 format files
16 """Processing unit to read HDF5 format files
17
17
18 This unit reads HDF5 files created with `HDFWriter` operation contains
18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 attributes.
20 attributes.
21 It is possible to read any HDF5 file by given the structure in the `description`
21 It is possible to read any HDF5 file by given the structure in the `description`
22 parameter, also you can add extra values to metadata with the parameter `extras`.
22 parameter, also you can add extra values to metadata with the parameter `extras`.
23
23
24 Parameters:
24 Parameters:
25 -----------
25 -----------
26 path : str
26 path : str
27 Path where files are located.
27 Path where files are located.
28 startDate : date
28 startDate : date
29 Start date of the files
29 Start date of the files
30 endDate : list
30 endDate : list
31 End date of the files
31 End date of the files
32 startTime : time
32 startTime : time
33 Start time of the files
33 Start time of the files
34 endTime : time
34 endTime : time
35 End time of the files
35 End time of the files
36 description : dict, optional
36 description : dict, optional
37 Dictionary with the description of the HDF5 file
37 Dictionary with the description of the HDF5 file
38 extras : dict, optional
38 extras : dict, optional
39 Dictionary with extra metadata to be be added to `dataOut`
39 Dictionary with extra metadata to be be added to `dataOut`
40
40
41 Examples
41 Examples
42 --------
42 --------
43
43
44 desc = {
44 desc = {
45 'Data': {
45 'Data': {
46 'data_output': ['u', 'v', 'w'],
46 'data_output': ['u', 'v', 'w'],
47 'utctime': 'timestamps',
47 'utctime': 'timestamps',
48 } ,
48 } ,
49 'Metadata': {
49 'Metadata': {
50 'heightList': 'heights'
50 'heightList': 'heights'
51 }
51 }
52 }
52 }
53
53
54 desc = {
54 desc = {
55 'Data': {
55 'Data': {
56 'data_output': 'winds',
56 'data_output': 'winds',
57 'utctime': 'timestamps'
57 'utctime': 'timestamps'
58 },
58 },
59 'Metadata': {
59 'Metadata': {
60 'heightList': 'heights'
60 'heightList': 'heights'
61 }
61 }
62 }
62 }
63
63
64 extras = {
64 extras = {
65 'timeZone': 300
65 'timeZone': 300
66 }
66 }
67
67
68 reader = project.addReadUnit(
68 reader = project.addReadUnit(
69 name='HDFReader',
69 name='HDFReader',
70 path='/path/to/files',
70 path='/path/to/files',
71 startDate='2019/01/01',
71 startDate='2019/01/01',
72 endDate='2019/01/31',
72 endDate='2019/01/31',
73 startTime='00:00:00',
73 startTime='00:00:00',
74 endTime='23:59:59',
74 endTime='23:59:59',
75 # description=json.dumps(desc),
75 # description=json.dumps(desc),
76 # extras=json.dumps(extras),
76 # extras=json.dumps(extras),
77 )
77 )
78
78
79 """
79 """
80
80
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82
82
83 def __init__(self):
83 def __init__(self):
84 ProcessingUnit.__init__(self)
84 ProcessingUnit.__init__(self)
85 self.dataOut = Parameters()
85 self.dataOut = Parameters()
86 self.ext = ".hdf5"
86 self.ext = ".hdf5"
87 self.optchar = "D"
87 self.optchar = "D"
88 self.meta = {}
88 self.meta = {}
89 self.data = {}
89 self.data = {}
90 self.open_file = h5py.File
90 self.open_file = h5py.File
91 self.open_mode = 'r'
91 self.open_mode = 'r'
92 self.description = {}
92 self.description = {}
93 self.extras = {}
93 self.extras = {}
94 self.filefmt = "*%Y%j***"
94 self.filefmt = "*%Y%j***"
95 self.folderfmt = "*%Y%j"
95 self.folderfmt = "*%Y%j"
96 self.utcoffset = 0
96
97
97 def setup(self, **kwargs):
98 def setup(self, **kwargs):
98
99
99 self.set_kwargs(**kwargs)
100 self.set_kwargs(**kwargs)
100 if not self.ext.startswith('.'):
101 if not self.ext.startswith('.'):
101 self.ext = '.{}'.format(self.ext)
102 self.ext = '.{}'.format(self.ext)
102
103
103 if self.online:
104 if self.online:
104 log.log("Searching files in online mode...", self.name)
105 log.log("Searching files in online mode...", self.name)
105
106
106 for nTries in range(self.nTries):
107 for nTries in range(self.nTries):
107 fullpath = self.searchFilesOnLine(self.path, self.startDate,
108 fullpath = self.searchFilesOnLine(self.path, self.startDate,
108 self.endDate, self.expLabel, self.ext, self.walk,
109 self.endDate, self.expLabel, self.ext, self.walk,
109 self.filefmt, self.folderfmt)
110 self.filefmt, self.folderfmt)
110 try:
111 try:
111 fullpath = next(fullpath)
112 fullpath = next(fullpath)
112 except:
113 except:
113 fullpath = None
114 fullpath = None
114
115
115 if fullpath:
116 if fullpath:
116 break
117 break
117
118
118 log.warning(
119 log.warning(
119 'Waiting {} sec for a valid file in {}: try {} ...'.format(
120 'Waiting {} sec for a valid file in {}: try {} ...'.format(
120 self.delay, self.path, nTries + 1),
121 self.delay, self.path, nTries + 1),
121 self.name)
122 self.name)
122 time.sleep(self.delay)
123 time.sleep(self.delay)
123
124
124 if not(fullpath):
125 if not(fullpath):
125 raise schainpy.admin.SchainError(
126 raise schainpy.admin.SchainError(
126 'There isn\'t any valid file in {}'.format(self.path))
127 'There isn\'t any valid file in {}'.format(self.path))
127
128
128 pathname, filename = os.path.split(fullpath)
129 pathname, filename = os.path.split(fullpath)
129 self.year = int(filename[1:5])
130 self.year = int(filename[1:5])
130 self.doy = int(filename[5:8])
131 self.doy = int(filename[5:8])
131 self.set = int(filename[8:11]) - 1
132 self.set = int(filename[8:11]) - 1
132 else:
133 else:
133 log.log("Searching files in {}".format(self.path), self.name)
134 log.log("Searching files in {}".format(self.path), self.name)
134 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
135 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
135 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
136 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
136
137
137 self.setNextFile()
138 self.setNextFile()
138
139
139 return
140 return
140
141
141 def readFirstHeader(self):
142 def readFirstHeader(self):
142 '''Read metadata and data'''
143 '''Read metadata and data'''
143
144
144 self.__readMetadata()
145 self.__readMetadata()
145 self.__readData()
146 self.__readData()
146 self.__setBlockList()
147 self.__setBlockList()
147
148
148 if 'type' in self.meta:
149 if 'type' in self.meta:
149 self.dataOut = eval(self.meta['type'])()
150 self.dataOut = eval(self.meta['type'])()
150
151
151 for attr in self.meta:
152 for attr in self.meta:
152 setattr(self.dataOut, attr, self.meta[attr])
153 setattr(self.dataOut, attr, self.meta[attr])
153
154
154 self.blockIndex = 0
155 self.blockIndex = 0
155
156
156 return
157 return
157
158
158 def __setBlockList(self):
159 def __setBlockList(self):
159 '''
160 '''
160 Selects the data within the times defined
161 Selects the data within the times defined
161
162
162 self.fp
163 self.fp
163 self.startTime
164 self.startTime
164 self.endTime
165 self.endTime
165 self.blockList
166 self.blockList
166 self.blocksPerFile
167 self.blocksPerFile
167
168
168 '''
169 '''
169
170
170 startTime = self.startTime
171 startTime = self.startTime
171 endTime = self.endTime
172 endTime = self.endTime
172
173 thisUtcTime = self.data['utctime'] + self.utcoffset
173 thisUtcTime = self.data['utctime']
174 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
174 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
175
176 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
175 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
177
176
178 thisDate = thisDatetime.date()
177 thisDate = thisDatetime.date()
179 thisTime = thisDatetime.time()
178 thisTime = thisDatetime.time()
180
179
181 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
180 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
182 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
181 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
183
182
184 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
183 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
185
184
186 self.blockList = ind
185 self.blockList = ind
187 self.blocksPerFile = len(ind)
186 self.blocksPerFile = len(ind)
188 return
187 return
189
188
190 def __readMetadata(self):
189 def __readMetadata(self):
191 '''
190 '''
192 Reads Metadata
191 Reads Metadata
193 '''
192 '''
194
193
195 meta = {}
194 meta = {}
196
195
197 if self.description:
196 if self.description:
198 for key, value in self.description['Metadata'].items():
197 for key, value in self.description['Metadata'].items():
199 meta[key] = self.fp[value][()]
198 meta[key] = self.fp[value][()]
200 else:
199 else:
201 grp = self.fp['Metadata']
200 grp = self.fp['Metadata']
202 for name in grp:
201 for name in grp:
203 meta[name] = grp[name][()]
202 meta[name] = grp[name][()]
204
203
205 if self.extras:
204 if self.extras:
206 for key, value in self.extras.items():
205 for key, value in self.extras.items():
207 meta[key] = value
206 meta[key] = value
208 self.meta = meta
207 self.meta = meta
209
208
210 return
209 return
211
210
212 def __readData(self):
211 def __readData(self):
213
212
214 data = {}
213 data = {}
215
214
216 if self.description:
215 if self.description:
217 for key, value in self.description['Data'].items():
216 for key, value in self.description['Data'].items():
218 if isinstance(value, str):
217 if isinstance(value, str):
219 if isinstance(self.fp[value], h5py.Dataset):
218 if isinstance(self.fp[value], h5py.Dataset):
220 data[key] = self.fp[value][()]
219 data[key] = self.fp[value][()]
221 elif isinstance(self.fp[value], h5py.Group):
220 elif isinstance(self.fp[value], h5py.Group):
222 array = []
221 array = []
223 for ch in self.fp[value]:
222 for ch in self.fp[value]:
224 array.append(self.fp[value][ch][()])
223 array.append(self.fp[value][ch][()])
225 data[key] = numpy.array(array)
224 data[key] = numpy.array(array)
226 elif isinstance(value, list):
225 elif isinstance(value, list):
227 array = []
226 array = []
228 for ch in value:
227 for ch in value:
229 array.append(self.fp[ch][()])
228 array.append(self.fp[ch][()])
230 data[key] = numpy.array(array)
229 data[key] = numpy.array(array)
231 else:
230 else:
232 grp = self.fp['Data']
231 grp = self.fp['Data']
233 for name in grp:
232 for name in grp:
234 if isinstance(grp[name], h5py.Dataset):
233 if isinstance(grp[name], h5py.Dataset):
235 array = grp[name][()]
234 array = grp[name][()]
236 elif isinstance(grp[name], h5py.Group):
235 elif isinstance(grp[name], h5py.Group):
237 array = []
236 array = []
238 for ch in grp[name]:
237 for ch in grp[name]:
239 array.append(grp[name][ch][()])
238 array.append(grp[name][ch][()])
240 array = numpy.array(array)
239 array = numpy.array(array)
241 else:
240 else:
242 log.warning('Unknown type: {}'.format(name))
241 log.warning('Unknown type: {}'.format(name))
243
242
244 if name in self.description:
243 if name in self.description:
245 key = self.description[name]
244 key = self.description[name]
246 else:
245 else:
247 key = name
246 key = name
248 data[key] = array
247 data[key] = array
249
248
250 self.data = data
249 self.data = data
251 return
250 return
252
251
253 def getData(self):
252 def getData(self):
254
253
255 for attr in self.data:
254 for attr in self.data:
256 if self.data[attr].ndim == 1:
255 if self.data[attr].ndim == 1:
257 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
256 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
258 else:
257 else:
259 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
258 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
260
259
261 self.dataOut.flagNoData = False
260 self.dataOut.flagNoData = False
262 self.blockIndex += 1
261 self.blockIndex += 1
263
262
264 log.log("Block No. {}/{} -> {}".format(
263 log.log("Block No. {}/{} -> {}".format(
265 self.blockIndex,
264 self.blockIndex,
266 self.blocksPerFile,
265 self.blocksPerFile,
267 self.dataOut.datatime.ctime()), self.name)
266 self.dataOut.datatime.ctime()), self.name)
268
267
269 return
268 return
270
269
271 def run(self, **kwargs):
270 def run(self, **kwargs):
272
271
273 if not(self.isConfig):
272 if not(self.isConfig):
274 self.setup(**kwargs)
273 self.setup(**kwargs)
275 self.isConfig = True
274 self.isConfig = True
276
275
277 if self.blockIndex == self.blocksPerFile:
276 if self.blockIndex == self.blocksPerFile:
278 self.setNextFile()
277 self.setNextFile()
279
278
280 self.getData()
279 self.getData()
281
280
282 return
281 return
283
282
284 @MPDecorator
283 @MPDecorator
285 class HDFWriter(Operation):
284 class HDFWriter(Operation):
286 """Operation to write HDF5 files.
285 """Operation to write HDF5 files.
287
286
288 The HDF5 file contains by default two groups Data and Metadata where
287 The HDF5 file contains by default two groups Data and Metadata where
289 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
288 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
290 parameters, data attributes are normaly time dependent where the metadata
289 parameters, data attributes are normaly time dependent where the metadata
291 are not.
290 are not.
292 It is possible to customize the structure of the HDF5 file with the
291 It is possible to customize the structure of the HDF5 file with the
293 optional description parameter see the examples.
292 optional description parameter see the examples.
294
293
295 Parameters:
294 Parameters:
296 -----------
295 -----------
297 path : str
296 path : str
298 Path where files will be saved.
297 Path where files will be saved.
299 blocksPerFile : int
298 blocksPerFile : int
300 Number of blocks per file
299 Number of blocks per file
301 metadataList : list
300 metadataList : list
302 List of the dataOut attributes that will be saved as metadata
301 List of the dataOut attributes that will be saved as metadata
303 dataList : int
302 dataList : int
304 List of the dataOut attributes that will be saved as data
303 List of the dataOut attributes that will be saved as data
305 setType : bool
304 setType : bool
306 If True the name of the files corresponds to the timestamp of the data
305 If True the name of the files corresponds to the timestamp of the data
307 description : dict, optional
306 description : dict, optional
308 Dictionary with the desired description of the HDF5 file
307 Dictionary with the desired description of the HDF5 file
309
308
310 Examples
309 Examples
311 --------
310 --------
312
311
313 desc = {
312 desc = {
314 'data_output': {'winds': ['z', 'w', 'v']},
313 'data_output': {'winds': ['z', 'w', 'v']},
315 'utctime': 'timestamps',
314 'utctime': 'timestamps',
316 'heightList': 'heights'
315 'heightList': 'heights'
317 }
316 }
318 desc = {
317 desc = {
319 'data_output': ['z', 'w', 'v'],
318 'data_output': ['z', 'w', 'v'],
320 'utctime': 'timestamps',
319 'utctime': 'timestamps',
321 'heightList': 'heights'
320 'heightList': 'heights'
322 }
321 }
323 desc = {
322 desc = {
324 'Data': {
323 'Data': {
325 'data_output': 'winds',
324 'data_output': 'winds',
326 'utctime': 'timestamps'
325 'utctime': 'timestamps'
327 },
326 },
328 'Metadata': {
327 'Metadata': {
329 'heightList': 'heights'
328 'heightList': 'heights'
330 }
329 }
331 }
330 }
332
331
333 writer = proc_unit.addOperation(name='HDFWriter')
332 writer = proc_unit.addOperation(name='HDFWriter')
334 writer.addParameter(name='path', value='/path/to/file')
333 writer.addParameter(name='path', value='/path/to/file')
335 writer.addParameter(name='blocksPerFile', value='32')
334 writer.addParameter(name='blocksPerFile', value='32')
336 writer.addParameter(name='metadataList', value='heightList,timeZone')
335 writer.addParameter(name='metadataList', value='heightList,timeZone')
337 writer.addParameter(name='dataList',value='data_output,utctime')
336 writer.addParameter(name='dataList',value='data_output,utctime')
338 # writer.addParameter(name='description',value=json.dumps(desc))
337 # writer.addParameter(name='description',value=json.dumps(desc))
339
338
340 """
339 """
341
340
342 ext = ".hdf5"
341 ext = ".hdf5"
343 optchar = "D"
342 optchar = "D"
344 filename = None
343 filename = None
345 path = None
344 path = None
346 setFile = None
345 setFile = None
347 fp = None
346 fp = None
348 firsttime = True
347 firsttime = True
349 #Configurations
348 #Configurations
350 blocksPerFile = None
349 blocksPerFile = None
351 blockIndex = None
350 blockIndex = None
352 dataOut = None
351 dataOut = None
353 #Data Arrays
352 #Data Arrays
354 dataList = None
353 dataList = None
355 metadataList = None
354 metadataList = None
356 currentDay = None
355 currentDay = None
357 lastTime = None
356 lastTime = None
358
357
359 def __init__(self):
358 def __init__(self):
360
359
361 Operation.__init__(self)
360 Operation.__init__(self)
362 return
361 return
363
362
364 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
363 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
365 self.path = path
364 self.path = path
366 self.blocksPerFile = blocksPerFile
365 self.blocksPerFile = blocksPerFile
367 self.metadataList = metadataList
366 self.metadataList = metadataList
368 self.dataList = [s.strip() for s in dataList]
367 self.dataList = [s.strip() for s in dataList]
369 self.setType = setType
368 self.setType = setType
370 self.description = description
369 self.description = description
371
370
372 if self.metadataList is None:
371 if self.metadataList is None:
373 self.metadataList = self.dataOut.metadata_list
372 self.metadataList = self.dataOut.metadata_list
374
373
375 tableList = []
374 tableList = []
376 dsList = []
375 dsList = []
377
376
378 for i in range(len(self.dataList)):
377 for i in range(len(self.dataList)):
379 dsDict = {}
378 dsDict = {}
380 if hasattr(self.dataOut, self.dataList[i]):
379 if hasattr(self.dataOut, self.dataList[i]):
381 dataAux = getattr(self.dataOut, self.dataList[i])
380 dataAux = getattr(self.dataOut, self.dataList[i])
382 dsDict['variable'] = self.dataList[i]
381 dsDict['variable'] = self.dataList[i]
383 else:
382 else:
384 log.warning('Attribute {} not found in dataOut', self.name)
383 log.warning('Attribute {} not found in dataOut', self.name)
385 continue
384 continue
386
385
387 if dataAux is None:
386 if dataAux is None:
388 continue
387 continue
389 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
388 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
390 dsDict['nDim'] = 0
389 dsDict['nDim'] = 0
391 else:
390 else:
392 dsDict['nDim'] = len(dataAux.shape)
391 dsDict['nDim'] = len(dataAux.shape)
393 dsDict['shape'] = dataAux.shape
392 dsDict['shape'] = dataAux.shape
394 dsDict['dsNumber'] = dataAux.shape[0]
393 dsDict['dsNumber'] = dataAux.shape[0]
395 dsDict['dtype'] = dataAux.dtype
394 dsDict['dtype'] = dataAux.dtype
396
395
397 dsList.append(dsDict)
396 dsList.append(dsDict)
398
397
399 self.dsList = dsList
398 self.dsList = dsList
400 self.currentDay = self.dataOut.datatime.date()
399 self.currentDay = self.dataOut.datatime.date()
401
400
402 def timeFlag(self):
401 def timeFlag(self):
403 currentTime = self.dataOut.utctime
402 currentTime = self.dataOut.utctime
404 timeTuple = time.localtime(currentTime)
403 timeTuple = time.localtime(currentTime)
405 dataDay = timeTuple.tm_yday
404 dataDay = timeTuple.tm_yday
406
405
407 if self.lastTime is None:
406 if self.lastTime is None:
408 self.lastTime = currentTime
407 self.lastTime = currentTime
409 self.currentDay = dataDay
408 self.currentDay = dataDay
410 return False
409 return False
411
410
412 timeDiff = currentTime - self.lastTime
411 timeDiff = currentTime - self.lastTime
413
412
414 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
413 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
415 if dataDay != self.currentDay:
414 if dataDay != self.currentDay:
416 self.currentDay = dataDay
415 self.currentDay = dataDay
417 return True
416 return True
418 elif timeDiff > 3*60*60:
417 elif timeDiff > 3*60*60:
419 self.lastTime = currentTime
418 self.lastTime = currentTime
420 return True
419 return True
421 else:
420 else:
422 self.lastTime = currentTime
421 self.lastTime = currentTime
423 return False
422 return False
424
423
425 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
424 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
426 dataList=[], setType=None, description={}):
425 dataList=[], setType=None, description={}):
427
426
428 self.dataOut = dataOut
427 self.dataOut = dataOut
429 if not(self.isConfig):
428 if not(self.isConfig):
430 self.setup(path=path, blocksPerFile=blocksPerFile,
429 self.setup(path=path, blocksPerFile=blocksPerFile,
431 metadataList=metadataList, dataList=dataList,
430 metadataList=metadataList, dataList=dataList,
432 setType=setType, description=description)
431 setType=setType, description=description)
433
432
434 self.isConfig = True
433 self.isConfig = True
435 self.setNextFile()
434 self.setNextFile()
436
435
437 self.putData()
436 self.putData()
438 return
437 return
439
438
440 def setNextFile(self):
439 def setNextFile(self):
441
440
442 ext = self.ext
441 ext = self.ext
443 path = self.path
442 path = self.path
444 setFile = self.setFile
443 setFile = self.setFile
445
444
446 timeTuple = time.localtime(self.dataOut.utctime)
445 timeTuple = time.localtime(self.dataOut.utctime)
447 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
446 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
448 fullpath = os.path.join(path, subfolder)
447 fullpath = os.path.join(path, subfolder)
449
448
450 if os.path.exists(fullpath):
449 if os.path.exists(fullpath):
451 filesList = os.listdir(fullpath)
450 filesList = os.listdir(fullpath)
452 filesList = [k for k in filesList if k.startswith(self.optchar)]
451 filesList = [k for k in filesList if k.startswith(self.optchar)]
453 if len( filesList ) > 0:
452 if len( filesList ) > 0:
454 filesList = sorted(filesList, key=str.lower)
453 filesList = sorted(filesList, key=str.lower)
455 filen = filesList[-1]
454 filen = filesList[-1]
456 # el filename debera tener el siguiente formato
455 # el filename debera tener el siguiente formato
457 # 0 1234 567 89A BCDE (hex)
456 # 0 1234 567 89A BCDE (hex)
458 # x YYYY DDD SSS .ext
457 # x YYYY DDD SSS .ext
459 if isNumber(filen[8:11]):
458 if isNumber(filen[8:11]):
460 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
459 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
461 else:
460 else:
462 setFile = -1
461 setFile = -1
463 else:
462 else:
464 setFile = -1 #inicializo mi contador de seteo
463 setFile = -1 #inicializo mi contador de seteo
465 else:
464 else:
466 os.makedirs(fullpath)
465 os.makedirs(fullpath)
467 setFile = -1 #inicializo mi contador de seteo
466 setFile = -1 #inicializo mi contador de seteo
468
467
469 if self.setType is None:
468 if self.setType is None:
470 setFile += 1
469 setFile += 1
471 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
470 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
472 timeTuple.tm_year,
471 timeTuple.tm_year,
473 timeTuple.tm_yday,
472 timeTuple.tm_yday,
474 setFile,
473 setFile,
475 ext )
474 ext )
476 else:
475 else:
477 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
476 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
478 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
477 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
479 timeTuple.tm_year,
478 timeTuple.tm_year,
480 timeTuple.tm_yday,
479 timeTuple.tm_yday,
481 setFile,
480 setFile,
482 ext )
481 ext )
483
482
484 self.filename = os.path.join( path, subfolder, file )
483 self.filename = os.path.join( path, subfolder, file )
485
484
486 #Setting HDF5 File
485 #Setting HDF5 File
487 self.fp = h5py.File(self.filename, 'w')
486 self.fp = h5py.File(self.filename, 'w')
488 #write metadata
487 #write metadata
489 self.writeMetadata(self.fp)
488 self.writeMetadata(self.fp)
490 #Write data
489 #Write data
491 self.writeData(self.fp)
490 self.writeData(self.fp)
492
491
493 def getLabel(self, name, x=None):
492 def getLabel(self, name, x=None):
494
493
495 if x is None:
494 if x is None:
496 if 'Data' in self.description:
495 if 'Data' in self.description:
497 data = self.description['Data']
496 data = self.description['Data']
498 if 'Metadata' in self.description:
497 if 'Metadata' in self.description:
499 data.update(self.description['Metadata'])
498 data.update(self.description['Metadata'])
500 else:
499 else:
501 data = self.description
500 data = self.description
502 if name in data:
501 if name in data:
503 if isinstance(data[name], str):
502 if isinstance(data[name], str):
504 return data[name]
503 return data[name]
505 elif isinstance(data[name], list):
504 elif isinstance(data[name], list):
506 return None
505 return None
507 elif isinstance(data[name], dict):
506 elif isinstance(data[name], dict):
508 for key, value in data[name].items():
507 for key, value in data[name].items():
509 return key
508 return key
510 return name
509 return name
511 else:
510 else:
512 if 'Metadata' in self.description:
511 if 'Metadata' in self.description:
513 meta = self.description['Metadata']
512 meta = self.description['Metadata']
514 else:
513 else:
515 meta = self.description
514 meta = self.description
516 if name in meta:
515 if name in meta:
517 if isinstance(meta[name], list):
516 if isinstance(meta[name], list):
518 return meta[name][x]
517 return meta[name][x]
519 elif isinstance(meta[name], dict):
518 elif isinstance(meta[name], dict):
520 for key, value in meta[name].items():
519 for key, value in meta[name].items():
521 return value[x]
520 return value[x]
522 if 'cspc' in name:
521 if 'cspc' in name:
523 return 'pair{:02d}'.format(x)
522 return 'pair{:02d}'.format(x)
524 else:
523 else:
525 return 'channel{:02d}'.format(x)
524 return 'channel{:02d}'.format(x)
526
525
527 def writeMetadata(self, fp):
526 def writeMetadata(self, fp):
528
527
529 if self.description:
528 if self.description:
530 if 'Metadata' in self.description:
529 if 'Metadata' in self.description:
531 grp = fp.create_group('Metadata')
530 grp = fp.create_group('Metadata')
532 else:
531 else:
533 grp = fp
532 grp = fp
534 else:
533 else:
535 grp = fp.create_group('Metadata')
534 grp = fp.create_group('Metadata')
536
535
537 for i in range(len(self.metadataList)):
536 for i in range(len(self.metadataList)):
538 if not hasattr(self.dataOut, self.metadataList[i]):
537 if not hasattr(self.dataOut, self.metadataList[i]):
539 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
538 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
540 continue
539 continue
541 value = getattr(self.dataOut, self.metadataList[i])
540 value = getattr(self.dataOut, self.metadataList[i])
542 if isinstance(value, bool):
541 if isinstance(value, bool):
543 if value is True:
542 if value is True:
544 value = 1
543 value = 1
545 else:
544 else:
546 value = 0
545 value = 0
547 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
546 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
548 return
547 return
549
548
550 def writeData(self, fp):
549 def writeData(self, fp):
551
550
552 if self.description:
551 if self.description:
553 if 'Data' in self.description:
552 if 'Data' in self.description:
554 grp = fp.create_group('Data')
553 grp = fp.create_group('Data')
555 else:
554 else:
556 grp = fp
555 grp = fp
557 else:
556 else:
558 grp = fp.create_group('Data')
557 grp = fp.create_group('Data')
559
558
560 dtsets = []
559 dtsets = []
561 data = []
560 data = []
562
561
563 for dsInfo in self.dsList:
562 for dsInfo in self.dsList:
564 if dsInfo['nDim'] == 0:
563 if dsInfo['nDim'] == 0:
565 ds = grp.create_dataset(
564 ds = grp.create_dataset(
566 self.getLabel(dsInfo['variable']),
565 self.getLabel(dsInfo['variable']),
567 (self.blocksPerFile, ),
566 (self.blocksPerFile, ),
568 chunks=True,
567 chunks=True,
569 dtype=numpy.float64)
568 dtype=numpy.float64)
570 dtsets.append(ds)
569 dtsets.append(ds)
571 data.append((dsInfo['variable'], -1))
570 data.append((dsInfo['variable'], -1))
572 else:
571 else:
573 label = self.getLabel(dsInfo['variable'])
572 label = self.getLabel(dsInfo['variable'])
574 if label is not None:
573 if label is not None:
575 sgrp = grp.create_group(label)
574 sgrp = grp.create_group(label)
576 else:
575 else:
577 sgrp = grp
576 sgrp = grp
578 for i in range(dsInfo['dsNumber']):
577 for i in range(dsInfo['dsNumber']):
579 ds = sgrp.create_dataset(
578 ds = sgrp.create_dataset(
580 self.getLabel(dsInfo['variable'], i),
579 self.getLabel(dsInfo['variable'], i),
581 (self.blocksPerFile, ) + dsInfo['shape'][1:],
580 (self.blocksPerFile, ) + dsInfo['shape'][1:],
582 chunks=True,
581 chunks=True,
583 dtype=dsInfo['dtype'])
582 dtype=dsInfo['dtype'])
584 dtsets.append(ds)
583 dtsets.append(ds)
585 data.append((dsInfo['variable'], i))
584 data.append((dsInfo['variable'], i))
586 fp.flush()
585 fp.flush()
587
586
588 log.log('Creating file: {}'.format(fp.filename), self.name)
587 log.log('Creating file: {}'.format(fp.filename), self.name)
589
588
590 self.ds = dtsets
589 self.ds = dtsets
591 self.data = data
590 self.data = data
592 self.firsttime = True
591 self.firsttime = True
593 self.blockIndex = 0
592 self.blockIndex = 0
594 return
593 return
595
594
596 def putData(self):
595 def putData(self):
597
596
598 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
597 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
599 self.closeFile()
598 self.closeFile()
600 self.setNextFile()
599 self.setNextFile()
601
600
602 for i, ds in enumerate(self.ds):
601 for i, ds in enumerate(self.ds):
603 attr, ch = self.data[i]
602 attr, ch = self.data[i]
604 if ch == -1:
603 if ch == -1:
605 ds[self.blockIndex] = getattr(self.dataOut, attr)
604 ds[self.blockIndex] = getattr(self.dataOut, attr)
606 else:
605 else:
607 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
606 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
608
607
609 self.fp.flush()
608 self.fp.flush()
610 self.blockIndex += 1
609 self.blockIndex += 1
611 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
610 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
612
611
613 return
612 return
614
613
615 def closeFile(self):
614 def closeFile(self):
616
615
617 if self.blockIndex != self.blocksPerFile:
616 if self.blockIndex != self.blocksPerFile:
618 for ds in self.ds:
617 for ds in self.ds:
619 ds.resize(self.blockIndex, axis=0)
618 ds.resize(self.blockIndex, axis=0)
620
619
621 if self.fp:
620 if self.fp:
622 self.fp.flush()
621 self.fp.flush()
623 self.fp.close()
622 self.fp.close()
624
623
625 def close(self):
624 def close(self):
626
625
627 self.closeFile()
626 self.closeFile()
General Comments 0
You need to be logged in to leave comments. Login now