##// END OF EJS Templates
schain
RSS
modified for reading channels with the same pointing
joabAM -
r1557:a1cf32393853
parent child
Show More
Side by Side Unified
Context file:
r1557:a1cf32393853 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,705 +1,705
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Base class to create plot operations
6 6
7 7 """
8 8
9 9 import os
10 10 import sys
11 11 import zmq
12 12 import time
13 13 import numpy
14 14 import datetime
15 15 from collections import deque
16 16 from functools import wraps
17 17 from threading import Thread
18 18 import matplotlib
19 19
20 20 if 'BACKEND' in os.environ:
21 21 matplotlib.use(os.environ['BACKEND'])
22 22 elif 'linux' in sys.platform:
23 23 matplotlib.use("TkAgg")
24 24 elif 'darwin' in sys.platform:
25 25 matplotlib.use('MacOSX')
26 26 else:
27 27 from schainpy.utils import log
28 28 log.warning('Using default Backend="Agg"', 'INFO')
29 29 matplotlib.use('Agg')
30 30
31 31 import matplotlib.pyplot as plt
32 32 from matplotlib.patches import Polygon
33 33 from mpl_toolkits.axes_grid1 import make_axes_locatable
34 34 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
35 35
36 36 from schainpy.model.data.jrodata import PlotterData
37 37 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
38 38 from schainpy.utils import log
39 39
40 40 jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
41 41 blu_values = matplotlib.pyplot.get_cmap(
42 42 'seismic_r', 20)(numpy.arange(20))[10:15]
43 43 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
44 44 'jro', numpy.vstack((blu_values, jet_values)))
45 45 matplotlib.pyplot.register_cmap(cmap=ncmap)
46 46
47 47 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis',
48 48 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
49 49
50 50 EARTH_RADIUS = 6.3710e3
51 51
52 52 def ll2xy(lat1, lon1, lat2, lon2):
53 53
54 54 p = 0.017453292519943295
55 55 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
56 56 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
57 57 r = 12742 * numpy.arcsin(numpy.sqrt(a))
58 58 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
59 59 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
60 60 theta = -theta + numpy.pi/2
61 61 return r*numpy.cos(theta), r*numpy.sin(theta)
62 62
63 63
64 64 def km2deg(km):
65 65 '''
66 66 Convert distance in km to degrees
67 67 '''
68 68
69 69 return numpy.rad2deg(km/EARTH_RADIUS)
70 70
71 71
72 72 def figpause(interval):
73 73 backend = plt.rcParams['backend']
74 74 if backend in matplotlib.rcsetup.interactive_bk:
75 75 figManager = matplotlib._pylab_helpers.Gcf.get_active()
76 76 if figManager is not None:
77 77 canvas = figManager.canvas
78 78 if canvas.figure.stale:
79 79 canvas.draw()
80 80 try:
81 81 canvas.start_event_loop(interval)
82 82 except:
83 83 pass
84 84 return
85 85
86 86 def popup(message):
87 87 '''
88 88 '''
89 89
90 90 fig = plt.figure(figsize=(12, 8), facecolor='r')
91 91 text = '\n'.join([s.strip() for s in message.split(':')])
92 92 fig.text(0.01, 0.5, text, ha='left', va='center',
93 93 size='20', weight='heavy', color='w')
94 94 fig.show()
95 95 figpause(1000)
96 96
97 97
98 98 class Throttle(object):
99 99 '''
100 100 Decorator that prevents a function from being called more than once every
101 101 time period.
102 102 To create a function that cannot be called more than once a minute, but
103 103 will sleep until it can be called:
104 104 @Throttle(minutes=1)
105 105 def foo():
106 106 pass
107 107
108 108 for i in range(10):
109 109 foo()
110 110 print "This function has run %s times." % i
111 111 '''
112 112
113 113 def __init__(self, seconds=0, minutes=0, hours=0):
114 114 self.throttle_period = datetime.timedelta(
115 115 seconds=seconds, minutes=minutes, hours=hours
116 116 )
117 117
118 118 self.time_of_last_call = datetime.datetime.min
119 119
120 120 def __call__(self, fn):
121 121 @wraps(fn)
122 122 def wrapper(*args, **kwargs):
123 123 coerce = kwargs.pop('coerce', None)
124 124 if coerce:
125 125 self.time_of_last_call = datetime.datetime.now()
126 126 return fn(*args, **kwargs)
127 127 else:
128 128 now = datetime.datetime.now()
129 129 time_since_last_call = now - self.time_of_last_call
130 130 time_left = self.throttle_period - time_since_last_call
131 131
132 132 if time_left > datetime.timedelta(seconds=0):
133 133 return
134 134
135 135 self.time_of_last_call = datetime.datetime.now()
136 136 return fn(*args, **kwargs)
137 137
138 138 return wrapper
139 139
140 140 def apply_throttle(value):
141 141
142 142 @Throttle(seconds=value)
143 143 def fnThrottled(fn):
144 144 fn()
145 145
146 146 return fnThrottled
147 147
148 148
149 149 @MPDecorator
150 150 class Plot(Operation):
151 151 """Base class for Schain plotting operations
152 152
153 153 This class should never be use directtly you must subclass a new operation,
154 154 children classes must be defined as follow:
155 155
156 156 ExamplePlot(Plot):
157 157
158 158 CODE = 'code'
159 159 colormap = 'jet'
160 160 plot_type = 'pcolor' # options are ('pcolor', 'pcolorbuffer', 'scatter', 'scatterbuffer')
161 161
162 162 def setup(self):
163 163 pass
164 164
165 165 def plot(self):
166 166 pass
167 167
168 168 """
169 169
170 170 CODE = 'Figure'
171 171 colormap = 'jet'
172 172 bgcolor = 'white'
173 173 buffering = True
174 174 __missing = 1E30
175 175
176 176 __attrs__ = ['show', 'save', 'ymin', 'ymax', 'zmin', 'zmax', 'title',
177 177 'showprofile']
178 178
179 179 def __init__(self):
180 180
181 181 Operation.__init__(self)
182 182 self.isConfig = False
183 183 self.isPlotConfig = False
184 184 self.save_time = 0
185 185 self.sender_time = 0
186 186 self.data = None
187 187 self.firsttime = True
188 188 self.sender_queue = deque(maxlen=10)
189 189 self.plots_adjust = {'left': 0.125, 'right': 0.9, 'bottom': 0.15, 'top': 0.9, 'wspace': 0.2, 'hspace': 0.2}
190 190
191 191 def __fmtTime(self, x, pos):
192 192 '''
193 193 '''
194 194 if self.t_units == "h_m":
195 195 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
196 196 if self.t_units == "h":
197 197 return '{}'.format(self.getDateTime(x).strftime('%H'))
198 198
199 199 def __setup(self, **kwargs):
200 200 '''
201 201 Initialize variables
202 202 '''
203 203
204 204 self.figures = []
205 205 self.axes = []
206 206 self.cb_axes = []
207 207 self.pf_axes = []
208 208 self.localtime = kwargs.pop('localtime', True)
209 209 self.show = kwargs.get('show', True)
210 210 self.save = kwargs.get('save', False)
211 211 self.save_period = kwargs.get('save_period', 0)
212 212 self.colormap = kwargs.get('colormap', self.colormap)
213 213 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
214 214 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
215 215 self.colormaps = kwargs.get('colormaps', None)
216 216 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
217 217 self.showprofile = kwargs.get('showprofile', False)
218 218 self.title = kwargs.get('wintitle', self.CODE.upper())
219 219 self.cb_label = kwargs.get('cb_label', None)
220 220 self.cb_labels = kwargs.get('cb_labels', None)
221 221 self.labels = kwargs.get('labels', None)
222 222 self.xaxis = kwargs.get('xaxis', 'frequency')
223 223 self.zmin = kwargs.get('zmin', None)
224 224 self.zmax = kwargs.get('zmax', None)
225 225 self.zlimits = kwargs.get('zlimits', None)
226 226 self.xmin = kwargs.get('xmin', None)
227 227 self.xmax = kwargs.get('xmax', None)
228 228 self.xrange = kwargs.get('xrange', 12)
229 229 self.xscale = kwargs.get('xscale', None)
230 230 self.ymin = kwargs.get('ymin', None)
231 231 self.ymax = kwargs.get('ymax', None)
232 232 self.yscale = kwargs.get('yscale', None)
233 233 self.xlabel = kwargs.get('xlabel', None)
234 234 self.attr_time = kwargs.get('attr_time', 'utctime')
235 235 self.attr_data = kwargs.get('attr_data', 'data_param')
236 236 self.decimation = kwargs.get('decimation', None)
237 237 self.oneFigure = kwargs.get('oneFigure', True)
238 238 self.width = kwargs.get('width', None)
239 239 self.height = kwargs.get('height', None)
240 240 self.colorbar = kwargs.get('colorbar', True)
241 241 self.factors = kwargs.get('factors', range(18))
242 242 self.channels = kwargs.get('channels', None)
243 243 self.titles = kwargs.get('titles', [])
244 244 self.polar = False
245 245 self.type = kwargs.get('type', 'iq')
246 246 self.grid = kwargs.get('grid', False)
247 247 self.pause = kwargs.get('pause', False)
248 248 self.save_code = kwargs.get('save_code', self.CODE)
249 249 self.throttle = kwargs.get('throttle', 0)
250 250 self.exp_code = kwargs.get('exp_code', None)
251 251 self.server = kwargs.get('server', False)
252 252 self.sender_period = kwargs.get('sender_period', 60)
253 253 self.tag = kwargs.get('tag', '')
254 254 self.height_index = kwargs.get('height_index', [])
255 255 self.__throttle_plot = apply_throttle(self.throttle)
256 256 code = self.attr_data if self.attr_data else self.CODE
257 257 self.data = PlotterData(self.CODE, self.exp_code, self.localtime)
258 258 self.tmin = kwargs.get('tmin', None)
259 259 self.t_units = kwargs.get('t_units', "h_m")
260 260 self.selectedHeightsList = kwargs.get('selectedHeightsList', [])
261 261 if isinstance(self.selectedHeightsList, int):
262 262 self.selectedHeightsList = [self.selectedHeightsList]
263 263
264 264 if self.server:
265 265 if not self.server.startswith('tcp://'):
266 266 self.server = 'tcp://{}'.format(self.server)
267 267 log.success(
268 268 'Sending to server: {}'.format(self.server),
269 269 self.name
270 270 )
271 271
272 272 if isinstance(self.attr_data, str):
273 273 self.attr_data = [self.attr_data]
274 274
275 275 def __setup_plot(self):
276 276 '''
277 277 Common setup for all figures, here figures and axes are created
278 278 '''
279 279
280 280 self.setup()
281 281
282 282 self.time_label = 'LT' if self.localtime else 'UTC'
283 283
284 284 if self.width is None:
285 285 self.width = 8
286 286
287 287 self.figures = []
288 288 self.axes = []
289 289 self.cb_axes = []
290 290 self.pf_axes = []
291 291 self.cmaps = []
292 292
293 293 size = '15%' if self.ncols == 1 else '30%'
294 294 pad = '4%' if self.ncols == 1 else '8%'
295 295
296 296 if self.oneFigure:
297 297 if self.height is None:
298 298 self.height = 1.4 * self.nrows + 1
299 299 fig = plt.figure(figsize=(self.width, self.height),
300 300 edgecolor='k',
301 301 facecolor='w')
302 302 self.figures.append(fig)
303 303 for n in range(self.nplots):
304 304 ax = fig.add_subplot(self.nrows, self.ncols,
305 305 n + 1, polar=self.polar)
306 306 ax.tick_params(labelsize=8)
307 307 ax.firsttime = True
308 308 ax.index = 0
309 309 ax.press = None
310 310 self.axes.append(ax)
311 311 if self.showprofile:
312 312 cax = self.__add_axes(ax, size=size, pad=pad)
313 313 cax.tick_params(labelsize=8)
314 314 self.pf_axes.append(cax)
315 315 else:
316 316 if self.height is None:
317 317 self.height = 3
318 318 for n in range(self.nplots):
319 319 fig = plt.figure(figsize=(self.width, self.height),
320 320 edgecolor='k',
321 321 facecolor='w')
322 322 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
323 323 ax.tick_params(labelsize=8)
324 324 ax.firsttime = True
325 325 ax.index = 0
326 326 ax.press = None
327 327 self.figures.append(fig)
328 328 self.axes.append(ax)
329 329 if self.showprofile:
330 330 cax = self.__add_axes(ax, size=size, pad=pad)
331 331 cax.tick_params(labelsize=8)
332 332 self.pf_axes.append(cax)
333 333
334 334 for n in range(self.nrows):
335 335 if self.colormaps is not None:
336 336 cmap = plt.get_cmap(self.colormaps[n])
337 337 else:
338 338 cmap = plt.get_cmap(self.colormap)
339 339 cmap.set_bad(self.bgcolor, 1.)
340 340 self.cmaps.append(cmap)
341 341
342 342 def __add_axes(self, ax, size='30%', pad='8%'):
343 343 '''
344 344 Add new axes to the given figure
345 345 '''
346 346 divider = make_axes_locatable(ax)
347 347 nax = divider.new_horizontal(size=size, pad=pad)
348 348 ax.figure.add_axes(nax)
349 349 return nax
350 350
351 351 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
352 352 '''
353 353 Create a masked array for missing data
354 354 '''
355 355 if x_buffer.shape[0] < 2:
356 356 return x_buffer, y_buffer, z_buffer
357 357
358 358 deltas = x_buffer[1:] - x_buffer[0:-1]
359 359 x_median = numpy.median(deltas)
360 360
361 361 index = numpy.where(deltas > 5 * x_median)
362 362
363 363 if len(index[0]) != 0:
364 364 z_buffer[::, index[0], ::] = self.__missing
365 365 z_buffer = numpy.ma.masked_inside(z_buffer,
366 366 0.99 * self.__missing,
367 367 1.01 * self.__missing)
368 368
369 369 return x_buffer, y_buffer, z_buffer
370 370
371 371 def decimate(self):
372 372
373 373 # dx = int(len(self.x)/self.__MAXNUMX) + 1
374 374 dy = int(len(self.y) / self.decimation) + 1
375 375
376 376 # x = self.x[::dx]
377 377 x = self.x
378 378 y = self.y[::dy]
379 379 z = self.z[::, ::, ::dy]
380 380
381 381 return x, y, z
382 382
383 383 def format(self):
384 384 '''
385 385 Set min and max values, labels, ticks and titles
386 386 '''
387 387
388 388 for n, ax in enumerate(self.axes):
389 389 if ax.firsttime:
390 390 if self.xaxis != 'time':
391 391 xmin = self.xmin
392 392 xmax = self.xmax
393 393 else:
394 394 xmin = self.tmin
395 395 xmax = self.tmin + self.xrange*60*60
396 396 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
397 397 if self.t_units == "h_m":
398 398 ax.xaxis.set_major_locator(LinearLocator(9))
399 399 if self.t_units == "h":
400 400 ax.xaxis.set_major_locator(LinearLocator(int((xmax-xmin)/3600)+1))
401 401 ymin = self.ymin if self.ymin is not None else numpy.nanmin(self.y[numpy.isfinite(self.y)])
402 402 ymax = self.ymax if self.ymax is not None else numpy.nanmax(self.y[numpy.isfinite(self.y)])
403 403 ax.set_facecolor(self.bgcolor)
404 404 if self.xscale:
405 405 ax.xaxis.set_major_formatter(FuncFormatter(
406 406 lambda x, pos: '{0:g}'.format(x*self.xscale)))
407 407 if self.yscale:
408 408 ax.yaxis.set_major_formatter(FuncFormatter(
409 409 lambda x, pos: '{0:g}'.format(x*self.yscale)))
410 410 if self.xlabel is not None:
411 411 ax.set_xlabel(self.xlabel)
412 412 if self.ylabel is not None:
413 413 ax.set_ylabel(self.ylabel)
414 414 if self.showprofile:
415 415 self.pf_axes[n].set_ylim(ymin, ymax)
416 416 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
417 417 self.pf_axes[n].set_xlabel('dB')
418 418 self.pf_axes[n].grid(b=True, axis='x')
419 419 [tick.set_visible(False)
420 420 for tick in self.pf_axes[n].get_yticklabels()]
421 421 if self.colorbar:
422 422 ax.cbar = plt.colorbar(
423 423 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
424 424 ax.cbar.ax.tick_params(labelsize=8)
425 425 ax.cbar.ax.press = None
426 426 if self.cb_label:
427 427 ax.cbar.set_label(self.cb_label, size=8)
428 428 elif self.cb_labels:
429 429 ax.cbar.set_label(self.cb_labels[n], size=8)
430 430 else:
431 431 ax.cbar = None
432 432 ax.set_xlim(xmin, xmax)
433 433 ax.set_ylim(ymin, ymax)
434 434 ax.firsttime = False
435 435 if self.grid:
436 436 ax.grid(True)
437 437
438 438 if not self.polar:
439 439 ax.set_title('{} {} {}'.format(
440 440 self.titles[n],
441 441 self.getDateTime(self.data.max_time).strftime(
442 442 '%Y-%m-%d %H:%M:%S'),
443 443 self.time_label),
444 444 size=8)
445 445 else:
446 446
447 447 ax.set_title('{}'.format(self.titles[n]), size=8)
448 448 ax.set_ylim(0, 90)
449 449 ax.set_yticks(numpy.arange(0, 90, 20))
450 450 ax.yaxis.labelpad = 40
451 451
452 452 if self.firsttime:
453 453 for n, fig in enumerate(self.figures):
454 454 fig.subplots_adjust(**self.plots_adjust)
455 455 self.firsttime = False
456 456
457 457 def clear_figures(self):
458 458 '''
459 459 Reset axes for redraw plots
460 460 '''
461 461
462 462 for ax in self.axes+self.pf_axes+self.cb_axes:
463 463 ax.clear()
464 464 ax.firsttime = True
465 465 if hasattr(ax, 'cbar') and ax.cbar:
466 466 ax.cbar.remove()
467 467
468 468 def __plot(self):
469 469 '''
470 470 Main function to plot, format and save figures
471 471 '''
472 472
473 473 self.plot()
474 474 self.format()
475 475
476 476 for n, fig in enumerate(self.figures):
477 477 if self.nrows == 0 or self.nplots == 0:
478 478 log.warning('No data', self.name)
479 479 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
480 480 fig.canvas.manager.set_window_title(self.CODE)
481 481 continue
482 482
483 483 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
484 484 self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
485 485
486 486 fig.canvas.draw()
487 487 if self.show:
488 488 fig.show()
489 489 figpause(0.01)
490 490
491 491 if self.save:
492 492 self.save_figure(n)
493 493
494 494 if self.server:
495 495 self.send_to_server()
496 496
497 497 def __update(self, dataOut, timestamp):
498 498 '''
499 499 '''
500 500
501 501 metadata = {
502 502 'yrange': dataOut.heightList,
503 503 'interval': dataOut.timeInterval,
504 504 'channels': dataOut.channelList
505 505 }
506 506 data, meta = self.update(dataOut)
507 507 metadata.update(meta)
508 508 self.data.update(data, timestamp, metadata)
509 509
510 510 def save_figure(self, n):
511 511 '''
512 512 '''
513 513
514 514 if (self.data.max_time - self.save_time) <= self.save_period:
515 515 return
516 516
517 517 self.save_time = self.data.max_time
518 518
519 519 fig = self.figures[n]
520 520
521 521 if self.throttle == 0:
522 522 figname = os.path.join(
523 523 self.save,
524 524 self.save_code,
525 '{}_{}.png'.format(
525 '{}_{}.jpeg'.format(
526 526 self.save_code,
527 527 self.getDateTime(self.data.max_time).strftime(
528 528 '%Y%m%d_%H%M%S'
529 529 ),
530 530 )
531 531 )
532 532 log.log('Saving figure: {}'.format(figname), self.name)
533 533 if not os.path.isdir(os.path.dirname(figname)):
534 534 os.makedirs(os.path.dirname(figname))
535 535 fig.savefig(figname)
536 536
537 537 figname = os.path.join(
538 538 self.save,
539 '{}_{}.png'.format(
539 '{}_{}.jpeg'.format(
540 540 self.save_code,
541 541 self.getDateTime(self.data.min_time).strftime(
542 542 '%Y%m%d'
543 543 ),
544 544 )
545 545 )
546 546
547 547 log.log('Saving figure: {}'.format(figname), self.name)
548 548 if not os.path.isdir(os.path.dirname(figname)):
549 549 os.makedirs(os.path.dirname(figname))
550 550 fig.savefig(figname)
551 551
552 552 def send_to_server(self):
553 553 '''
554 554 '''
555 555
556 556 if self.exp_code == None:
557 557 log.warning('Missing `exp_code` skipping sending to server...')
558 558
559 559 last_time = self.data.max_time
560 560 interval = last_time - self.sender_time
561 561 if interval < self.sender_period:
562 562 return
563 563
564 564 self.sender_time = last_time
565 565
566 566 attrs = ['titles', 'zmin', 'zmax', 'tag', 'ymin', 'ymax']
567 567 for attr in attrs:
568 568 value = getattr(self, attr)
569 569 if value:
570 570 if isinstance(value, (numpy.float32, numpy.float64)):
571 571 value = round(float(value), 2)
572 572 self.data.meta[attr] = value
573 573 if self.colormap == 'jet':
574 574 self.data.meta['colormap'] = 'Jet'
575 575 elif 'RdBu' in self.colormap:
576 576 self.data.meta['colormap'] = 'RdBu'
577 577 else:
578 578 self.data.meta['colormap'] = 'Viridis'
579 579 self.data.meta['interval'] = int(interval)
580 580
581 581 self.sender_queue.append(last_time)
582 582
583 583 while 1:
584 584 try:
585 585 tm = self.sender_queue.popleft()
586 586 except IndexError:
587 587 break
588 588 msg = self.data.jsonify(tm, self.save_code, self.plot_type)
589 589 self.socket.send_string(msg)
590 590 socks = dict(self.poll.poll(2000))
591 591 if socks.get(self.socket) == zmq.POLLIN:
592 592 reply = self.socket.recv_string()
593 593 if reply == 'ok':
594 594 log.log("Response from server ok", self.name)
595 595 time.sleep(0.1)
596 596 continue
597 597 else:
598 598 log.warning(
599 599 "Malformed reply from server: {}".format(reply), self.name)
600 600 else:
601 601 log.warning(
602 602 "No response from server, retrying...", self.name)
603 603 self.sender_queue.appendleft(tm)
604 604 self.socket.setsockopt(zmq.LINGER, 0)
605 605 self.socket.close()
606 606 self.poll.unregister(self.socket)
607 607 self.socket = self.context.socket(zmq.REQ)
608 608 self.socket.connect(self.server)
609 609 self.poll.register(self.socket, zmq.POLLIN)
610 610 break
611 611
612 612 def setup(self):
613 613 '''
614 614 This method should be implemented in the child class, the following
615 615 attributes should be set:
616 616
617 617 self.nrows: number of rows
618 618 self.ncols: number of cols
619 619 self.nplots: number of plots (channels or pairs)
620 620 self.ylabel: label for Y axes
621 621 self.titles: list of axes title
622 622
623 623 '''
624 624 raise NotImplementedError
625 625
626 626 def plot(self):
627 627 '''
628 628 Must be defined in the child class, the actual plotting method
629 629 '''
630 630 raise NotImplementedError
631 631
632 632 def update(self, dataOut):
633 633 '''
634 634 Must be defined in the child class, update self.data with new data
635 635 '''
636 636
637 637 data = {
638 638 self.CODE: getattr(dataOut, 'data_{}'.format(self.CODE))
639 639 }
640 640 meta = {}
641 641
642 642 return data, meta
643 643
644 644 def run(self, dataOut, **kwargs):
645 645 '''
646 646 Main plotting routine
647 647 '''
648 648 if self.isConfig is False:
649 649 self.__setup(**kwargs)
650 650
651 651 if self.localtime:
652 652 self.getDateTime = datetime.datetime.fromtimestamp
653 653 else:
654 654 self.getDateTime = datetime.datetime.utcfromtimestamp
655 655
656 656 self.data.setup()
657 657 self.isConfig = True
658 658 if self.server:
659 659 self.context = zmq.Context()
660 660 self.socket = self.context.socket(zmq.REQ)
661 661 self.socket.connect(self.server)
662 662 self.poll = zmq.Poller()
663 663 self.poll.register(self.socket, zmq.POLLIN)
664 664
665 665 tm = getattr(dataOut, self.attr_time)
666 666
667 667 if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60:
668 668 self.save_time = tm
669 669 self.__plot()
670 670 self.tmin += self.xrange*60*60
671 671 self.data.setup()
672 672 self.clear_figures()
673 673
674 674 self.__update(dataOut, tm)
675 675
676 676 if self.isPlotConfig is False:
677 677 self.__setup_plot()
678 678 self.isPlotConfig = True
679 679 if self.xaxis == 'time':
680 680 dt = self.getDateTime(tm)
681 681 if self.xmin is None:
682 682 self.tmin = tm
683 683 self.xmin = dt.hour
684 684 minutes = (self.xmin-int(self.xmin)) * 60
685 685 seconds = (minutes - int(minutes)) * 60
686 686 self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) -
687 687 datetime.datetime(1970, 1, 1)).total_seconds()
688 688 if self.localtime:
689 689 self.tmin += time.timezone
690 690
691 691 if self.xmin is not None and self.xmax is not None:
692 692 self.xrange = self.xmax - self.xmin
693 693
694 694 if self.throttle == 0:
695 695 self.__plot()
696 696 else:
697 697 self.__throttle_plot(self.__plot)#, coerce=coerce)
698 698
699 699 def close(self):
700 700
701 701 if self.data and not self.data.flagNoData:
702 702 self.save_time = 0
703 703 self.__plot()
704 704 if self.data and not self.data.flagNoData and self.pause:
705 705 figpause(10)
Show file before | Show file after | Hide comments
@@ -1,671 +1,691
1 1 ''''
2 2 Created on Set 9, 2015
3 3
4 4 @author: roj-idl71 Karim Kuyeng
5 5
6 6 @update: 2021, Joab Apaza
7 7 '''
8 8
9 9 import os
10 10 import sys
11 11 import glob
12 12 import fnmatch
13 13 import datetime
14 14 import time
15 15 import re
16 16 import h5py
17 17 import numpy
18 18
19 19 try:
20 20 from gevent import sleep
21 21 except:
22 22 from time import sleep
23 23
24 24 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
25 25 from schainpy.model.data.jrodata import Voltage
26 26 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
27 27 from numpy import imag
28 28 from schainpy.utils import log
29 29
30 30
31 31 class AMISRReader(ProcessingUnit):
32 32 '''
33 33 classdocs
34 34 '''
35 35
36 36 def __init__(self):
37 37 '''
38 38 Constructor
39 39 '''
40 40
41 41 ProcessingUnit.__init__(self)
42 42
43 43 self.set = None
44 44 self.subset = None
45 45 self.extension_file = '.h5'
46 46 self.dtc_str = 'dtc'
47 47 self.dtc_id = 0
48 48 self.status = True
49 49 self.isConfig = False
50 50 self.dirnameList = []
51 51 self.filenameList = []
52 52 self.fileIndex = None
53 53 self.flagNoMoreFiles = False
54 54 self.flagIsNewFile = 0
55 55 self.filename = ''
56 56 self.amisrFilePointer = None
57 self.realBeamCode = []
57
58 58 self.beamCodeMap = None
59 59 self.azimuthList = []
60 60 self.elevationList = []
61 61 self.dataShape = None
62 62 self.flag_old_beams = False
63 63
64 64
65 65 self.profileIndex = 0
66 66
67 67
68 68 self.beamCodeByFrame = None
69 69 self.radacTimeByFrame = None
70 70
71 71 self.dataset = None
72 72
73 73 self.__firstFile = True
74 74
75 75 self.buffer = None
76 76
77 77 self.timezone = 'ut'
78 78
79 79 self.__waitForNewFile = 20
80 80 self.__filename_online = None
81 81 #Is really necessary create the output object in the initializer
82 82 self.dataOut = Voltage()
83 83 self.dataOut.error=False
84 84 self.margin_days = 1
85 85
86 86 def setup(self,path=None,
87 87 startDate=None,
88 88 endDate=None,
89 89 startTime=None,
90 90 endTime=None,
91 91 walk=True,
92 92 timezone='ut',
93 93 all=0,
94 94 code = None,
95 95 nCode = 1,
96 96 nBaud = 0,
97 97 online=False,
98 98 old_beams=False,
99 margin_days=1):
99 margin_days=1,
100 nFFT = 1,
101 nChannels = None,
102 ):
100 103
101 104
102 105
103 106 self.timezone = timezone
104 107 self.all = all
105 108 self.online = online
106 109 self.flag_old_beams = old_beams
107 110 self.code = code
108 111 self.nCode = int(nCode)
109 112 self.nBaud = int(nBaud)
110 113 self.margin_days = margin_days
111 114 self.__sampleRate = None
112 115
116 self.nFFT = nFFT
117 self.nChannels = nChannels
118
113 119 #self.findFiles()
114 120 if not(online):
115 121 #Busqueda de archivos offline
116 122 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
117 123 else:
118 124 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
119 125
120 126 if not(self.filenameList):
121 127 raise schainpy.admin.SchainWarning("There is no files into the folder: %s"%(path))
122 128 #sys.exit(0)
123 129 self.dataOut.error = True
124 130
125 131 self.fileIndex = 0
126 132
127 133 self.readNextFile(online)
128 134
129 135 '''
130 136 Add code
131 137 '''
132 138 self.isConfig = True
133 139 # print("Setup Done")
134 140 pass
135 141
136 142
137 143 def readAMISRHeader(self,fp):
138 144
139 145 if self.isConfig and (not self.flagNoMoreFiles):
140 146 newShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
141 147 if self.dataShape != newShape and newShape != None:
142 148 raise schainpy.admin.SchainError("NEW FILE HAS A DIFFERENT SHAPE: ")
143 149 print(self.dataShape,newShape,"\n")
144 150 return 0
145 151 else:
146 152 self.dataShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
147 153
148 154
149 155 header = 'Raw11/Data/RadacHeader'
156 if self.nChannels == None:
157 expFile = fp['Setup/ExperimentFile'][()].decode()
158 linesExp = expFile.split("\n")
159 a = [line for line in linesExp if "nbeamcodes" in line]
160 self.nChannels = int(a[0][11:])
161
150 162 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
151 if (self.startDate> datetime.date(2021, 7, 15)) or self.flag_old_beams: #Se cambiΓ³ la forma de extracciΓ³n de Apuntes el 17 o forzar con flag de reorganizaciΓ³n
163 if (self.startDate > datetime.date(2021, 7, 15)) or self.flag_old_beams: #Se cambiΓ³ la forma de extracciΓ³n de Apuntes el 17 o forzar con flag de reorganizaciΓ³n
152 164 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
153 165 self.trueBeams = self.beamcodeFile.split("\n")
154 166 self.trueBeams.pop()#remove last
155 [self.realBeamCode.append(x) for x in self.trueBeams if x not in self.realBeamCode]
156 self.beamCode = [int(x, 16) for x in self.realBeamCode]
167 beams_idx = [k*self.nFFT for k in range(self.nChannels)]
168 beams = [self.trueBeams[b] for b in beams_idx]
169 self.beamCode = [int(x, 16) for x in beams]
170
157 171 else:
158 172 _beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
159 173 self.beamCode = _beamCode[0,:]
160 174
161 175 if self.beamCodeMap == None:
162 176 self.beamCodeMap = fp['Setup/BeamcodeMap']
163 177 for beam in self.beamCode:
164 178 beamAziElev = numpy.where(self.beamCodeMap[:,0]==beam)
165 179 beamAziElev = beamAziElev[0].squeeze()
166 180 self.azimuthList.append(self.beamCodeMap[beamAziElev,1])
167 181 self.elevationList.append(self.beamCodeMap[beamAziElev,2])
168 182 #print("Beamssss: ",self.beamCodeMap[beamAziElev,1],self.beamCodeMap[beamAziElev,2])
169 183 #print(self.beamCode)
170 184 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
171 185 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
172 186 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
173 187 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
174 188 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
175 189 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
176 190 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
177 191 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
178 192 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
179 193 self.frequency = fp.get('Rx/Frequency')
180 194 txAus = fp.get('Raw11/Data/Pulsewidth')
181 195 self.baud = fp.get('Raw11/Data/TxBaud')
182 196 sampleRate = fp.get('Rx/SampleRate')
183 197 self.__sampleRate = sampleRate[()]
184 198 self.nblocks = self.pulseCount.shape[0] #nblocks
185
199 self.profPerBlockRAW = self.pulseCount.shape[1] #profiles per block in raw data
186 200 self.nprofiles = self.pulseCount.shape[1] #nprofile
187 201 self.nsa = self.nsamplesPulse[0,0] #ngates
188 202 self.nchannels = len(self.beamCode)
189 203 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
190 204 #print("IPPS secs: ",self.ippSeconds)
191 205 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
192 206 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
193 207
194 208 #filling radar controller header parameters
195 209 self.__ippKm = self.ippSeconds *.15*1e6 # in km
196 210 self.__txA = (txAus[()])*.15 #(ipp[us]*.15km/1us) in km
197 211 self.__txB = 0
198 212 nWindows=1
199 213 self.__nSamples = self.nsa
200 214 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
201 215 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
202 216 #print("amisr-ipp:",self.ippSeconds, self.__ippKm)
203 217 #for now until understand why the code saved is different (code included even though code not in tuf file)
204 218 #self.__codeType = 0
205 219 # self.__nCode = None
206 220 # self.__nBaud = None
207 221 self.__code = self.code
208 222 self.__codeType = 0
209 223 if self.code != None:
210 224 self.__codeType = 1
211 225 self.__nCode = self.nCode
212 226 self.__nBaud = self.nBaud
213 227 #self.__code = 0
214 228
215 229 #filling system header parameters
216 230 self.__nSamples = self.nsa
217 231 self.newProfiles = self.nprofiles/self.nchannels
218 232 self.__channelList = [n for n in range(self.nchannels)]
219 233
220 234 self.__frequency = self.frequency[0][0]
221 235
222 236
223 237 return 1
224 238
225 239
226 240 def createBuffers(self):
227 241
228 242 pass
229 243
230 244 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
231 245 self.path = path
232 246 self.startDate = startDate
233 247 self.endDate = endDate
234 248 self.startTime = startTime
235 249 self.endTime = endTime
236 250 self.walk = walk
237 251
238 252 def __checkPath(self):
239 253 if os.path.exists(self.path):
240 254 self.status = 1
241 255 else:
242 256 self.status = 0
243 257 print('Path:%s does not exists'%self.path)
244 258
245 259 return
246 260
247 261
248 262 def __selDates(self, amisr_dirname_format):
249 263 try:
250 264 year = int(amisr_dirname_format[0:4])
251 265 month = int(amisr_dirname_format[4:6])
252 266 dom = int(amisr_dirname_format[6:8])
253 267 thisDate = datetime.date(year,month,dom)
254 268 #margen de un dΓ­a extra, igual luego se filtra for fecha y hora
255 269 if (thisDate>=(self.startDate - datetime.timedelta(days=self.margin_days)) and thisDate <= (self.endDate)+ datetime.timedelta(days=1)):
256 270 return amisr_dirname_format
257 271 except:
258 272 return None
259 273
260 274
261 275 def __findDataForDates(self,online=False):
262 276
263 277 if not(self.status):
264 278 return None
265 279
266 280 pat = '\d+.\d+'
267 281 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
268 282 dirnameList = [x for x in dirnameList if x!=None]
269 283 dirnameList = [x.string for x in dirnameList]
270 284 if not(online):
271 285 dirnameList = [self.__selDates(x) for x in dirnameList]
272 286 dirnameList = [x for x in dirnameList if x!=None]
273 287 if len(dirnameList)>0:
274 288 self.status = 1
275 289 self.dirnameList = dirnameList
276 290 self.dirnameList.sort()
277 291 else:
278 292 self.status = 0
279 293 return None
280 294
281 295 def __getTimeFromData(self):
282 296 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
283 297 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
284 298
285 299 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
286 300 print('........................................')
287 301 filter_filenameList = []
288 302 self.filenameList.sort()
289 303 total_files = len(self.filenameList)
290 304 #for i in range(len(self.filenameList)-1):
291 305 for i in range(total_files):
292 306 filename = self.filenameList[i]
293 307 #print("file-> ",filename)
294 308 try:
295 309 fp = h5py.File(filename,'r')
296 310 time_str = fp.get('Time/RadacTimeString')
297 311
298 312 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
299 313 #startDateTimeStr_File = "2019-12-16 09:21:11"
300 314 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
301 315 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
302 316
303 317 #endDateTimeStr_File = "2019-12-16 11:10:11"
304 318 endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
305 319 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
306 320 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
307 321
308 322 fp.close()
309 323
310 324 #print("check time", startDateTime_File)
311 325 if self.timezone == 'lt':
312 326 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
313 327 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
314 328 if (startDateTime_File >=startDateTime_Reader and endDateTime_File<=endDateTime_Reader):
315 329 filter_filenameList.append(filename)
316 330
317 331 if (startDateTime_File>endDateTime_Reader):
318 332 break
319 333 except Exception as e:
320 334 log.warning("Error opening file {} -> {}".format(os.path.split(filename)[1],e))
321 335
322 336 filter_filenameList.sort()
323 337 self.filenameList = filter_filenameList
324 338
325 339 return 1
326 340
327 341 def __filterByGlob1(self, dirName):
328 342 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
329 343 filter_files.sort()
330 344 filterDict = {}
331 345 filterDict.setdefault(dirName)
332 346 filterDict[dirName] = filter_files
333 347 return filterDict
334 348
335 349 def __getFilenameList(self, fileListInKeys, dirList):
336 350 for value in fileListInKeys:
337 351 dirName = list(value.keys())[0]
338 352 for file in value[dirName]:
339 353 filename = os.path.join(dirName, file)
340 354 self.filenameList.append(filename)
341 355
342 356
343 357 def __selectDataForTimes(self, online=False):
344 358 #aun no esta implementado el filtro for tiempo
345 359 if not(self.status):
346 360 return None
347 361
348 362 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
349 363 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
350 364 self.__getFilenameList(fileListInKeys, dirList)
351 365 if not(online):
352 366 #filtro por tiempo
353 367 if not(self.all):
354 368 self.__getTimeFromData()
355 369
356 370 if len(self.filenameList)>0:
357 371 self.status = 1
358 372 self.filenameList.sort()
359 373 else:
360 374 self.status = 0
361 375 return None
362 376
363 377 else:
364 378 #get the last file - 1
365 379 self.filenameList = [self.filenameList[-2]]
366 380 new_dirnameList = []
367 381 for dirname in self.dirnameList:
368 382 junk = numpy.array([dirname in x for x in self.filenameList])
369 383 junk_sum = junk.sum()
370 384 if junk_sum > 0:
371 385 new_dirnameList.append(dirname)
372 386 self.dirnameList = new_dirnameList
373 387 return 1
374 388
375 389 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
376 390 endTime=datetime.time(23,59,59),walk=True):
377 391
378 392 if endDate ==None:
379 393 startDate = datetime.datetime.utcnow().date()
380 394 endDate = datetime.datetime.utcnow().date()
381 395
382 396 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
383 397
384 398 self.__checkPath()
385 399
386 400 self.__findDataForDates(online=True)
387 401
388 402 self.dirnameList = [self.dirnameList[-1]]
389 403
390 404 self.__selectDataForTimes(online=True)
391 405
392 406 return
393 407
394 408
395 409 def searchFilesOffLine(self,
396 410 path,
397 411 startDate,
398 412 endDate,
399 413 startTime=datetime.time(0,0,0),
400 414 endTime=datetime.time(23,59,59),
401 415 walk=True):
402 416
403 417 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
404 418
405 419 self.__checkPath()
406 420
407 421 self.__findDataForDates()
408 422
409 423 self.__selectDataForTimes()
410 424
411 425 for i in range(len(self.filenameList)):
412 426 print("%s" %(self.filenameList[i]))
413 427
414 428 return
415 429
416 430 def __setNextFileOffline(self):
417 431
418 432 try:
419 433 self.filename = self.filenameList[self.fileIndex]
420 434 self.amisrFilePointer = h5py.File(self.filename,'r')
421 435 self.fileIndex += 1
422 436 except:
423 437 self.flagNoMoreFiles = 1
424 438 raise schainpy.admin.SchainError('No more files to read')
425 439 return 0
426 440
427 441 self.flagIsNewFile = 1
428 442 print("Setting the file: %s"%self.filename)
429 443
430 444 return 1
431 445
432 446
433 447 def __setNextFileOnline(self):
434 448 filename = self.filenameList[0]
435 449 if self.__filename_online != None:
436 450 self.__selectDataForTimes(online=True)
437 451 filename = self.filenameList[0]
438 452 wait = 0
439 453 self.__waitForNewFile=300 ## DEBUG:
440 454 while self.__filename_online == filename:
441 455 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
442 456 if wait == 5:
443 457 self.flagNoMoreFiles = 1
444 458 return 0
445 459 sleep(self.__waitForNewFile)
446 460 self.__selectDataForTimes(online=True)
447 461 filename = self.filenameList[0]
448 462 wait += 1
449 463
450 464 self.__filename_online = filename
451 465
452 466 self.amisrFilePointer = h5py.File(filename,'r')
453 467 self.flagIsNewFile = 1
454 468 self.filename = filename
455 469 print("Setting the file: %s"%self.filename)
456 470 return 1
457 471
458 472
459 473 def readData(self):
460 474 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
461 475 re = buffer[:,:,:,0]
462 476 im = buffer[:,:,:,1]
463 477 dataset = re + im*1j
464 478
465 479 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
466 480 timeset = self.radacTime[:,0]
467 481
468 482 return dataset,timeset
469 483
470 484 def reshapeData(self):
471 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
485 #print(self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa)
472 486 channels = self.beamCodeByPulse[0,:]
473 487 nchan = self.nchannels
474 488 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
475 489 nblocks = self.nblocks
476 490 nsamples = self.nsa
477
491 #print("Channels: ",self.nChannels)
478 492 #Dimensions : nChannels, nProfiles, nSamples
479 493 new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
480 494 ############################################
495 profPerCH = int(self.profPerBlockRAW / (self.nFFT * self.nChannels))
481 496
482 497 for thisChannel in range(nchan):
483 new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[thisChannel])[0],:]
484 498
499 idx_ch = [thisChannel+k*nchan for k in range(profPerCH)]
500 idx_ch = numpy.array(idx_ch, dtype=int)
501 #print(thisChannel,idx_ch)
502 #print(numpy.where(channels==self.beamCode[thisChannel])[0])
503 #new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[thisChannel])[0],:]
504 new_block[:,thisChannel,:,:] = self.dataset[:,idx_ch,:]
485 505
486 506 new_block = numpy.transpose(new_block, (1,0,2,3))
487 507 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
488 508
489 509 return new_block
490 510
491 511 def updateIndexes(self):
492 512
493 513 pass
494 514
495 515 def fillJROHeader(self):
496 516
497 517 #fill radar controller header
498 518 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
499 519 txA=self.__txA,
500 520 txB=0,
501 521 nWindows=1,
502 522 nHeights=self.__nSamples,
503 523 firstHeight=self.__firstHeight,
504 524 deltaHeight=self.__deltaHeight,
505 525 codeType=self.__codeType,
506 526 nCode=self.__nCode, nBaud=self.__nBaud,
507 527 code = self.__code,
508 528 fClock=self.__sampleRate)
509 529 #fill system header
510 530 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
511 531 nProfiles=self.newProfiles,
512 532 nChannels=len(self.__channelList),
513 533 adcResolution=14,
514 534 pciDioBusWidth=32)
515 535
516 536 self.dataOut.type = "Voltage"
517 537 self.dataOut.data = None
518 538 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
519 539 # self.dataOut.nChannels = 0
520 540
521 541 # self.dataOut.nHeights = 0
522 542
523 543 self.dataOut.nProfiles = self.newProfiles*self.nblocks
524 544 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
525 545 ranges = numpy.reshape(self.rangeFromFile[()],(-1))
526 546 self.dataOut.heightList = ranges/1000.0 #km
527 547 self.dataOut.channelList = self.__channelList
528 548 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
529 549
530 550 # self.dataOut.channelIndexList = None
531 551
532 552
533 553 self.dataOut.azimuthList = numpy.array(self.azimuthList)
534 554 self.dataOut.elevationList = numpy.array(self.elevationList)
535 555 self.dataOut.codeList = numpy.array(self.beamCode)
536 556 #print(self.dataOut.elevationList)
537 557 self.dataOut.flagNoData = True
538 558
539 559 #Set to TRUE if the data is discontinuous
540 560 self.dataOut.flagDiscontinuousBlock = False
541 561
542 562 self.dataOut.utctime = None
543 563
544 564 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
545 565 if self.timezone == 'lt':
546 566 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
547 567 else:
548 568 self.dataOut.timeZone = 0 #by default time is UTC
549 569
550 570 self.dataOut.dstFlag = 0
551 571 self.dataOut.errorCount = 0
552 572 self.dataOut.nCohInt = 1
553 573 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
554 574 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
555 575 self.dataOut.flagShiftFFT = False
556 576 self.dataOut.ippSeconds = self.ippSeconds
557 577 self.dataOut.radar_ipp = self.ippSeconds
558 578 self.dataOut.pulseLength_TxA = self.__txA/0.15
559 579 self.dataOut.deltaHeight = self.__deltaHeight
560 580 #Time interval between profiles
561 581 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
562 582
563 583 self.dataOut.frequency = self.__frequency
564 584 self.dataOut.realtime = self.online
565 585 pass
566 586
567 587 def readNextFile(self,online=False):
568 588
569 589 if not(online):
570 590 newFile = self.__setNextFileOffline()
571 591 else:
572 592 newFile = self.__setNextFileOnline()
573 593
574 594 if not(newFile):
575 595 self.dataOut.error = True
576 596 return 0
577 597
578 598 if not self.readAMISRHeader(self.amisrFilePointer):
579 599 self.dataOut.error = True
580 600 return 0
581 601
582 602 self.createBuffers()
583 603 self.fillJROHeader()
584 604
585 605 #self.__firstFile = False
586 606
587 607
588 608
589 609 self.dataset,self.timeset = self.readData()
590 610
591 611 if self.endDate!=None:
592 612 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
593 613 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
594 614 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
595 615 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
596 616 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
597 617 if self.timezone == 'lt':
598 618 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
599 619 if (startDateTime_File>endDateTime_Reader):
600 620 return 0
601 621
602 622 self.jrodataset = self.reshapeData()
603 623 #----self.updateIndexes()
604 624 self.profileIndex = 0
605 625
606 626 return 1
607 627
608 628
609 629 def __hasNotDataInBuffer(self):
610 630 if self.profileIndex >= (self.newProfiles*self.nblocks):
611 631 return 1
612 632 return 0
613 633
614 634
615 635 def getData(self):
616 636
617 637 if self.flagNoMoreFiles:
618 638 self.dataOut.flagNoData = True
619 639 return 0
620 640
621 641 if self.profileIndex >= (self.newProfiles*self.nblocks): #
622 642 #if self.__hasNotDataInBuffer():
623 643 if not (self.readNextFile(self.online)):
624 644 return 0
625 645
626 646
627 647 if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
628 648 self.dataOut.flagNoData = True
629 649 return 0
630 650
631 651 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
632 652
633 653 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
634 654
635 655 #print("R_t",self.timeset)
636 656
637 657 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
638 658 #verificar basic header de jro data y ver si es compatible con este valor
639 659 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
640 660 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
641 661 indexblock = self.profileIndex/self.newProfiles
642 662 #print (indexblock, indexprof)
643 663 diffUTC = 0
644 664 t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
645 665
646 666 #print("utc :",indexblock," __ ",t_comp)
647 667 #print(numpy.shape(self.timeset))
648 668 self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
649 669 #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
650 670
651 671 self.dataOut.profileIndex = self.profileIndex
652 672 #print("N profile:",self.profileIndex,self.newProfiles,self.nblocks,self.dataOut.utctime)
653 673 self.dataOut.flagNoData = False
654 674 # if indexprof == 0:
655 675 # print("kamisr: ",self.dataOut.utctime)
656 676
657 677 self.profileIndex += 1
658 678
659 679 return self.dataOut.data #retorno necesario??
660 680
661 681
662 682 def run(self, **kwargs):
663 683 '''
664 684 This method will be called many times so here you should put all your code
665 685 '''
666 686 #print("running kamisr")
667 687 if not self.isConfig:
668 688 self.setup(**kwargs)
669 689 self.isConfig = True
670 690
671 691 self.getData()
General Comments 0
You need to be logged in to leave comments. Login now