##// END OF EJS Templates
schain-jc
RSS
Fix PXReader for online data, add date to plot's title
jespinoza -
r1143:9ee641cc2d04
parent child
Show More
Side by Side Unified
Context file:
r1143:9ee641cc2d04 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1085 +1,1085
1 1
2 2 import os
3 3 import time
4 4 import glob
5 5 import datetime
6 6 from multiprocessing import Process
7 7
8 8 import zmq
9 9 import numpy
10 10 import matplotlib
11 11 import matplotlib.pyplot as plt
12 12 from mpl_toolkits.axes_grid1 import make_axes_locatable
13 13 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
14 14
15 15 from schainpy.model.proc.jroproc_base import Operation
16 16 from schainpy.utils import log
17 17
18 18 jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
19 19 blu_values = matplotlib.pyplot.get_cmap(
20 20 'seismic_r', 20)(numpy.arange(20))[10:15]
21 21 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
22 22 'jro', numpy.vstack((blu_values, jet_values)))
23 23 matplotlib.pyplot.register_cmap(cmap=ncmap)
24 24
25 25 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis', 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm', 'spectral')]
26 26
27 27
28 28 def figpause(interval):
29 29 backend = plt.rcParams['backend']
30 30 if backend in matplotlib.rcsetup.interactive_bk:
31 31 figManager = matplotlib._pylab_helpers.Gcf.get_active()
32 32 if figManager is not None:
33 33 canvas = figManager.canvas
34 34 if canvas.figure.stale:
35 35 canvas.draw()
36 36 canvas.start_event_loop(interval)
37 37 return
38 38
39 39
40 40 class PlotData(Operation, Process):
41 41 '''
42 42 Base class for Schain plotting operations
43 43 '''
44 44
45 45 CODE = 'Figure'
46 46 colormap = 'jro'
47 47 bgcolor = 'white'
48 48 CONFLATE = False
49 49 __missing = 1E30
50 50
51 51 __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
52 52 'zlimits', 'xlabel', 'ylabel', 'xaxis','cb_label', 'title',
53 53 'colorbar', 'bgcolor', 'width', 'height', 'localtime', 'oneFigure',
54 54 'showprofile', 'decimation', 'ftp']
55 55
56 56 def __init__(self, **kwargs):
57 57
58 58 Operation.__init__(self, plot=True, **kwargs)
59 59 Process.__init__(self)
60 60
61 61 self.kwargs['code'] = self.CODE
62 62 self.mp = False
63 63 self.data = None
64 64 self.isConfig = False
65 65 self.figures = []
66 66 self.axes = []
67 67 self.cb_axes = []
68 68 self.localtime = kwargs.pop('localtime', True)
69 69 self.show = kwargs.get('show', True)
70 70 self.save = kwargs.get('save', False)
71 71 self.ftp = kwargs.get('ftp', False)
72 72 self.colormap = kwargs.get('colormap', self.colormap)
73 73 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
74 74 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
75 75 self.colormaps = kwargs.get('colormaps', None)
76 76 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
77 77 self.showprofile = kwargs.get('showprofile', False)
78 78 self.title = kwargs.get('wintitle', self.CODE.upper())
79 79 self.cb_label = kwargs.get('cb_label', None)
80 80 self.cb_labels = kwargs.get('cb_labels', None)
81 81 self.labels = kwargs.get('labels', None)
82 82 self.xaxis = kwargs.get('xaxis', 'frequency')
83 83 self.zmin = kwargs.get('zmin', None)
84 84 self.zmax = kwargs.get('zmax', None)
85 85 self.zlimits = kwargs.get('zlimits', None)
86 86 self.xmin = kwargs.get('xmin', None)
87 87 self.xmax = kwargs.get('xmax', None)
88 88 self.xrange = kwargs.get('xrange', 24)
89 89 self.xscale = kwargs.get('xscale', None)
90 90 self.ymin = kwargs.get('ymin', None)
91 91 self.ymax = kwargs.get('ymax', None)
92 92 self.yscale = kwargs.get('yscale', None)
93 93 self.xlabel = kwargs.get('xlabel', None)
94 94 self.decimation = kwargs.get('decimation', None)
95 95 self.showSNR = kwargs.get('showSNR', False)
96 96 self.oneFigure = kwargs.get('oneFigure', True)
97 97 self.width = kwargs.get('width', None)
98 98 self.height = kwargs.get('height', None)
99 99 self.colorbar = kwargs.get('colorbar', True)
100 100 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
101 101 self.channels = kwargs.get('channels', None)
102 102 self.titles = kwargs.get('titles', [])
103 103 self.polar = False
104 104
105 105 def __fmtTime(self, x, pos):
106 106 '''
107 107 '''
108 108
109 109 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
110 110
111 111 def __setup(self):
112 112 '''
113 113 Common setup for all figures, here figures and axes are created
114 114 '''
115 115
116 116 if self.CODE not in self.data:
117 117 raise ValueError(log.error('Missing data for {}'.format(self.CODE),
118 118 self.name))
119 119
120 120 self.setup()
121 121
122 122 self.time_label = 'LT' if self.localtime else 'UTC'
123 123 if self.data.localtime:
124 124 self.getDateTime = datetime.datetime.fromtimestamp
125 125 else:
126 126 self.getDateTime = datetime.datetime.utcfromtimestamp
127 127
128 128 if self.width is None:
129 129 self.width = 8
130 130
131 131 self.figures = []
132 132 self.axes = []
133 133 self.cb_axes = []
134 134 self.pf_axes = []
135 135 self.cmaps = []
136 136
137 137 size = '15%' if self.ncols == 1 else '30%'
138 138 pad = '4%' if self.ncols == 1 else '8%'
139 139
140 140 if self.oneFigure:
141 141 if self.height is None:
142 142 self.height = 1.4 * self.nrows + 1
143 143 fig = plt.figure(figsize=(self.width, self.height),
144 144 edgecolor='k',
145 145 facecolor='w')
146 146 self.figures.append(fig)
147 147 for n in range(self.nplots):
148 148 ax = fig.add_subplot(self.nrows, self.ncols,
149 149 n + 1, polar=self.polar)
150 150 ax.tick_params(labelsize=8)
151 151 ax.firsttime = True
152 152 ax.index = 0
153 153 ax.press = None
154 154 self.axes.append(ax)
155 155 if self.showprofile:
156 156 cax = self.__add_axes(ax, size=size, pad=pad)
157 157 cax.tick_params(labelsize=8)
158 158 self.pf_axes.append(cax)
159 159 else:
160 160 if self.height is None:
161 161 self.height = 3
162 162 for n in range(self.nplots):
163 163 fig = plt.figure(figsize=(self.width, self.height),
164 164 edgecolor='k',
165 165 facecolor='w')
166 166 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
167 167 ax.tick_params(labelsize=8)
168 168 ax.firsttime = True
169 169 ax.index = 0
170 170 ax.press = None
171 171 self.figures.append(fig)
172 172 self.axes.append(ax)
173 173 if self.showprofile:
174 174 cax = self.__add_axes(ax, size=size, pad=pad)
175 175 cax.tick_params(labelsize=8)
176 176 self.pf_axes.append(cax)
177 177
178 178 for n in range(self.nrows):
179 179 if self.colormaps is not None:
180 180 cmap = plt.get_cmap(self.colormaps[n])
181 181 else:
182 182 cmap = plt.get_cmap(self.colormap)
183 183 cmap.set_bad(self.bgcolor, 1.)
184 184 self.cmaps.append(cmap)
185 185
186 186 for fig in self.figures:
187 187 fig.canvas.mpl_connect('key_press_event', self.OnKeyPress)
188 188 fig.canvas.mpl_connect('scroll_event', self.OnBtnScroll)
189 189 fig.canvas.mpl_connect('button_press_event', self.onBtnPress)
190 190 fig.canvas.mpl_connect('motion_notify_event', self.onMotion)
191 191 fig.canvas.mpl_connect('button_release_event', self.onBtnRelease)
192 192 if self.show:
193 193 fig.show()
194 194
195 195 def OnKeyPress(self, event):
196 196 '''
197 197 Event for pressing keys (up, down) change colormap
198 198 '''
199 199 ax = event.inaxes
200 200 if ax in self.axes:
201 201 if event.key == 'down':
202 202 ax.index += 1
203 203 elif event.key == 'up':
204 204 ax.index -= 1
205 205 if ax.index < 0:
206 206 ax.index = len(CMAPS) - 1
207 207 elif ax.index == len(CMAPS):
208 208 ax.index = 0
209 209 cmap = CMAPS[ax.index]
210 210 ax.cbar.set_cmap(cmap)
211 211 ax.cbar.draw_all()
212 212 ax.plt.set_cmap(cmap)
213 213 ax.cbar.patch.figure.canvas.draw()
214 214 self.colormap = cmap.name
215 215
216 216 def OnBtnScroll(self, event):
217 217 '''
218 218 Event for scrolling, scale figure
219 219 '''
220 220 cb_ax = event.inaxes
221 221 if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
222 222 ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
223 223 pt = ax.cbar.ax.bbox.get_points()[:, 1]
224 224 nrm = ax.cbar.norm
225 225 vmin, vmax, p0, p1, pS = (
226 226 nrm.vmin, nrm.vmax, pt[0], pt[1], event.y)
227 227 scale = 2 if event.step == 1 else 0.5
228 228 point = vmin + (vmax - vmin) / (p1 - p0) * (pS - p0)
229 229 ax.cbar.norm.vmin = point - scale * (point - vmin)
230 230 ax.cbar.norm.vmax = point - scale * (point - vmax)
231 231 ax.plt.set_norm(ax.cbar.norm)
232 232 ax.cbar.draw_all()
233 233 ax.cbar.patch.figure.canvas.draw()
234 234
235 235 def onBtnPress(self, event):
236 236 '''
237 237 Event for mouse button press
238 238 '''
239 239 cb_ax = event.inaxes
240 240 if cb_ax is None:
241 241 return
242 242
243 243 if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
244 244 cb_ax.press = event.x, event.y
245 245 else:
246 246 cb_ax.press = None
247 247
248 248 def onMotion(self, event):
249 249 '''
250 250 Event for move inside colorbar
251 251 '''
252 252 cb_ax = event.inaxes
253 253 if cb_ax is None:
254 254 return
255 255 if cb_ax not in [ax.cbar.ax for ax in self.axes if ax.cbar]:
256 256 return
257 257 if cb_ax.press is None:
258 258 return
259 259
260 260 ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
261 261 xprev, yprev = cb_ax.press
262 262 dx = event.x - xprev
263 263 dy = event.y - yprev
264 264 cb_ax.press = event.x, event.y
265 265 scale = ax.cbar.norm.vmax - ax.cbar.norm.vmin
266 266 perc = 0.03
267 267
268 268 if event.button == 1:
269 269 ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
270 270 ax.cbar.norm.vmax -= (perc * scale) * numpy.sign(dy)
271 271 elif event.button == 3:
272 272 ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
273 273 ax.cbar.norm.vmax += (perc * scale) * numpy.sign(dy)
274 274
275 275 ax.cbar.draw_all()
276 276 ax.plt.set_norm(ax.cbar.norm)
277 277 ax.cbar.patch.figure.canvas.draw()
278 278
279 279 def onBtnRelease(self, event):
280 280 '''
281 281 Event for mouse button release
282 282 '''
283 283 cb_ax = event.inaxes
284 284 if cb_ax is not None:
285 285 cb_ax.press = None
286 286
287 287 def __add_axes(self, ax, size='30%', pad='8%'):
288 288 '''
289 289 Add new axes to the given figure
290 290 '''
291 291 divider = make_axes_locatable(ax)
292 292 nax = divider.new_horizontal(size=size, pad=pad)
293 293 ax.figure.add_axes(nax)
294 294 return nax
295 295
296 296 self.setup()
297 297
298 298 def setup(self):
299 299 '''
300 300 This method should be implemented in the child class, the following
301 301 attributes should be set:
302 302
303 303 self.nrows: number of rows
304 304 self.ncols: number of cols
305 305 self.nplots: number of plots (channels or pairs)
306 306 self.ylabel: label for Y axes
307 307 self.titles: list of axes title
308 308
309 309 '''
310 310 raise(NotImplementedError, 'Implement this method in child class')
311 311
312 312 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
313 313 '''
314 314 Create a masked array for missing data
315 315 '''
316 316 if x_buffer.shape[0] < 2:
317 317 return x_buffer, y_buffer, z_buffer
318 318
319 319 deltas = x_buffer[1:] - x_buffer[0:-1]
320 320 x_median = numpy.median(deltas)
321 321
322 322 index = numpy.where(deltas > 5 * x_median)
323 323
324 324 if len(index[0]) != 0:
325 325 z_buffer[::, index[0], ::] = self.__missing
326 326 z_buffer = numpy.ma.masked_inside(z_buffer,
327 327 0.99 * self.__missing,
328 328 1.01 * self.__missing)
329 329
330 330 return x_buffer, y_buffer, z_buffer
331 331
332 332 def decimate(self):
333 333
334 334 # dx = int(len(self.x)/self.__MAXNUMX) + 1
335 335 dy = int(len(self.y) / self.decimation) + 1
336 336
337 337 # x = self.x[::dx]
338 338 x = self.x
339 339 y = self.y[::dy]
340 340 z = self.z[::, ::, ::dy]
341 341
342 342 return x, y, z
343 343
344 344 def format(self):
345 345 '''
346 346 Set min and max values, labels, ticks and titles
347 347 '''
348 348
349 349 if self.xmin is None:
350 350 xmin = self.min_time
351 351 else:
352 352 if self.xaxis is 'time':
353 353 dt = self.getDateTime(self.min_time)
354 354 xmin = (dt.replace(hour=int(self.xmin), minute=0, second=0) -
355 355 datetime.datetime(1970, 1, 1)).total_seconds()
356 356 if self.data.localtime:
357 357 xmin += time.timezone
358 358 else:
359 359 xmin = self.xmin
360 360
361 361 if self.xmax is None:
362 362 xmax = xmin + self.xrange * 60 * 60
363 363 else:
364 364 if self.xaxis is 'time':
365 365 dt = self.getDateTime(self.max_time)
366 366 xmax = (dt.replace(hour=int(self.xmax), minute=59, second=59) -
367 367 datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=1)).total_seconds()
368 368 if self.data.localtime:
369 369 xmax += time.timezone
370 370 else:
371 371 xmax = self.xmax
372 372
373 373 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
374 374 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
375 375
376 376 Y = numpy.array([5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000])
377 377 i = 1 if numpy.where(abs(ymax-ymin) <= Y)[0][0] < 0 else numpy.where(abs(ymax-ymin) <= Y)[0][0]
378 378 ystep = Y[i] / 5.
379 379
380 380 for n, ax in enumerate(self.axes):
381 381 if ax.firsttime:
382 382 ax.set_facecolor(self.bgcolor)
383 383 ax.yaxis.set_major_locator(MultipleLocator(ystep))
384 384 ax.xaxis.set_major_locator(MultipleLocator(ystep))
385 385 if self.xscale:
386 386 ax.xaxis.set_major_formatter(FuncFormatter(lambda x, pos: '{0:g}'.format(x*self.xscale)))
387 387 if self.xscale:
388 388 ax.yaxis.set_major_formatter(FuncFormatter(lambda x, pos: '{0:g}'.format(x*self.yscale)))
389 389 if self.xaxis is 'time':
390 390 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
391 391 ax.xaxis.set_major_locator(LinearLocator(9))
392 392 if self.xlabel is not None:
393 393 ax.set_xlabel(self.xlabel)
394 394 ax.set_ylabel(self.ylabel)
395 395 ax.firsttime = False
396 396 if self.showprofile:
397 397 self.pf_axes[n].set_ylim(ymin, ymax)
398 398 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
399 399 self.pf_axes[n].set_xlabel('dB')
400 400 self.pf_axes[n].grid(b=True, axis='x')
401 401 [tick.set_visible(False)
402 402 for tick in self.pf_axes[n].get_yticklabels()]
403 403 if self.colorbar:
404 404 ax.cbar = plt.colorbar(
405 405 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
406 406 ax.cbar.ax.tick_params(labelsize=8)
407 407 ax.cbar.ax.press = None
408 408 if self.cb_label:
409 409 ax.cbar.set_label(self.cb_label, size=8)
410 410 elif self.cb_labels:
411 411 ax.cbar.set_label(self.cb_labels[n], size=8)
412 412 else:
413 413 ax.cbar = None
414 414
415 415 if not self.polar:
416 416 ax.set_xlim(xmin, xmax)
417 417 ax.set_ylim(ymin, ymax)
418 ax.set_title('{} - {} {}'.format(
418 ax.set_title('{} {} {}'.format(
419 419 self.titles[n],
420 self.getDateTime(self.max_time).strftime('%H:%M:%S'),
420 self.getDateTime(self.max_time).strftime('%Y-%m-%dT%H:%M:%S'),
421 421 self.time_label),
422 422 size=8)
423 423 else:
424 424 ax.set_title('{}'.format(self.titles[n]), size=8)
425 425 ax.set_ylim(0, 90)
426 426 ax.set_yticks(numpy.arange(0, 90, 20))
427 427 ax.yaxis.labelpad = 40
428 428
429 429 def __plot(self):
430 430 '''
431 431 '''
432 432 log.log('Plotting', self.name)
433 433
434 434 try:
435 435 self.plot()
436 436 self.format()
437 437 except Exception as e:
438 438 log.warning('{} Plot could not be updated... check data'.format(self.CODE), self.name)
439 439 log.error(str(e), '')
440 440 return
441 441
442 442 for n, fig in enumerate(self.figures):
443 443 if self.nrows == 0 or self.nplots == 0:
444 444 log.warning('No data', self.name)
445 445 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
446 446 fig.canvas.manager.set_window_title(self.CODE)
447 447 continue
448 448
449 449 fig.tight_layout()
450 450 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
451 451 self.getDateTime(self.max_time).strftime('%Y/%m/%d')))
452 452 fig.canvas.draw()
453 453
454 454 if self.save and (self.data.ended or not self.data.buffering):
455 455
456 456 if self.save_labels:
457 457 labels = self.save_labels
458 458 else:
459 459 labels = range(self.nrows)
460 460
461 461 if self.oneFigure:
462 462 label = ''
463 463 else:
464 464 label = '-{}'.format(labels[n])
465 465 figname = os.path.join(
466 466 self.save,
467 467 self.CODE,
468 468 '{}{}_{}.png'.format(
469 469 self.CODE,
470 470 label,
471 471 self.getDateTime(self.saveTime).strftime(
472 472 '%Y%m%d_%H%M%S'),
473 473 )
474 474 )
475 475 log.log('Saving figure: {}'.format(figname), self.name)
476 476 if not os.path.isdir(os.path.dirname(figname)):
477 477 os.makedirs(os.path.dirname(figname))
478 478 fig.savefig(figname)
479 479
480 480 def plot(self):
481 481 '''
482 482 '''
483 483 raise(NotImplementedError, 'Implement this method in child class')
484 484
485 485 def run(self):
486 486
487 487 log.log('Starting', self.name)
488 488
489 489 context = zmq.Context()
490 490 receiver = context.socket(zmq.SUB)
491 491 receiver.setsockopt(zmq.SUBSCRIBE, '')
492 492 receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
493 493
494 494 if 'server' in self.kwargs['parent']:
495 495 receiver.connect(
496 496 'ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
497 497 else:
498 498 receiver.connect("ipc:///tmp/zmq.plots")
499 499
500 500 while True:
501 501 try:
502 502 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
503 503 if self.data.localtime and self.localtime:
504 504 self.times = self.data.times
505 505 elif self.data.localtime and not self.localtime:
506 506 self.times = self.data.times + time.timezone
507 507 elif not self.data.localtime and self.localtime:
508 508 self.times = self.data.times - time.timezone
509 509 else:
510 510 self.times = self.data.times
511 511
512 512 self.min_time = self.times[0]
513 513 self.max_time = self.times[-1]
514 514
515 515 if self.isConfig is False:
516 516 self.__setup()
517 517 self.isConfig = True
518 518
519 519 self.__plot()
520 520
521 521 except zmq.Again as e:
522 522 # log.log('.', tag='', nl=False)
523 523 if self.data:
524 524 figpause(self.data.throttle)
525 525 else:
526 526 time.sleep(2)
527 527
528 528 def close(self):
529 529 if self.data:
530 530 self.__plot()
531 531
532 532
533 533 class PlotSpectraData(PlotData):
534 534 '''
535 535 Plot for Spectra data
536 536 '''
537 537
538 538 CODE = 'spc'
539 539 colormap = 'jro'
540 540
541 541 def setup(self):
542 542 self.nplots = len(self.data.channels)
543 543 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
544 544 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
545 545 self.width = 3.4 * self.ncols
546 546 self.height = 3 * self.nrows
547 547 self.cb_label = 'dB'
548 548 if self.showprofile:
549 549 self.width += 0.8 * self.ncols
550 550
551 551 self.ylabel = 'Range [km]'
552 552
553 553 def plot(self):
554 554 if self.xaxis == "frequency":
555 555 x = self.data.xrange[0]
556 556 self.xlabel = "Frequency (kHz)"
557 557 elif self.xaxis == "time":
558 558 x = self.data.xrange[1]
559 559 self.xlabel = "Time (ms)"
560 560 else:
561 561 x = self.data.xrange[2]
562 562 self.xlabel = "Velocity (m/s)"
563 563
564 564 if self.CODE == 'spc_mean':
565 565 x = self.data.xrange[2]
566 566 self.xlabel = "Velocity (m/s)"
567 567
568 568 self.titles = []
569 569
570 570 y = self.data.heights
571 571 self.y = y
572 572 z = self.data['spc']
573 573
574 574 for n, ax in enumerate(self.axes):
575 575 noise = self.data['noise'][n][-1]
576 576 if self.CODE == 'spc_mean':
577 577 mean = self.data['mean'][n][-1]
578 578 if ax.firsttime:
579 579 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
580 580 self.xmin = self.xmin if self.xmin else -self.xmax
581 581 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
582 582 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
583 583 ax.plt = ax.pcolormesh(x, y, z[n].T,
584 584 vmin=self.zmin,
585 585 vmax=self.zmax,
586 586 cmap=plt.get_cmap(self.colormap)
587 587 )
588 588
589 589 if self.showprofile:
590 590 ax.plt_profile = self.pf_axes[n].plot(
591 591 self.data['rti'][n][-1], y)[0]
592 592 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
593 593 color="k", linestyle="dashed", lw=1)[0]
594 594 if self.CODE == 'spc_mean':
595 595 ax.plt_mean = ax.plot(mean, y, color='k')[0]
596 596 else:
597 597 ax.plt.set_array(z[n].T.ravel())
598 598 if self.showprofile:
599 599 ax.plt_profile.set_data(self.data['rti'][n][-1], y)
600 600 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
601 601 if self.CODE == 'spc_mean':
602 602 ax.plt_mean.set_data(mean, y)
603 603
604 604 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
605 605 self.saveTime = self.max_time
606 606
607 607
608 608 class PlotCrossSpectraData(PlotData):
609 609
610 610 CODE = 'cspc'
611 611 zmin_coh = None
612 612 zmax_coh = None
613 613 zmin_phase = None
614 614 zmax_phase = None
615 615
616 616 def setup(self):
617 617
618 618 self.ncols = 4
619 619 self.nrows = len(self.data.pairs)
620 620 self.nplots = self.nrows * 4
621 621 self.width = 3.4 * self.ncols
622 622 self.height = 3 * self.nrows
623 623 self.ylabel = 'Range [km]'
624 624 self.showprofile = False
625 625
626 626 def plot(self):
627 627
628 628 if self.xaxis == "frequency":
629 629 x = self.data.xrange[0]
630 630 self.xlabel = "Frequency (kHz)"
631 631 elif self.xaxis == "time":
632 632 x = self.data.xrange[1]
633 633 self.xlabel = "Time (ms)"
634 634 else:
635 635 x = self.data.xrange[2]
636 636 self.xlabel = "Velocity (m/s)"
637 637
638 638 self.titles = []
639 639
640 640 y = self.data.heights
641 641 self.y = y
642 642 spc = self.data['spc']
643 643 cspc = self.data['cspc']
644 644
645 645 for n in range(self.nrows):
646 646 noise = self.data['noise'][n][-1]
647 647 pair = self.data.pairs[n]
648 648 ax = self.axes[4 * n]
649 649 ax3 = self.axes[4 * n + 3]
650 650 if ax.firsttime:
651 651 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
652 652 self.xmin = self.xmin if self.xmin else -self.xmax
653 653 self.zmin = self.zmin if self.zmin else numpy.nanmin(spc)
654 654 self.zmax = self.zmax if self.zmax else numpy.nanmax(spc)
655 655 ax.plt = ax.pcolormesh(x, y, spc[pair[0]].T,
656 656 vmin=self.zmin,
657 657 vmax=self.zmax,
658 658 cmap=plt.get_cmap(self.colormap)
659 659 )
660 660 else:
661 661 ax.plt.set_array(spc[pair[0]].T.ravel())
662 662 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
663 663
664 664 ax = self.axes[4 * n + 1]
665 665 if ax.firsttime:
666 666 ax.plt = ax.pcolormesh(x, y, spc[pair[1]].T,
667 667 vmin=self.zmin,
668 668 vmax=self.zmax,
669 669 cmap=plt.get_cmap(self.colormap)
670 670 )
671 671 else:
672 672 ax.plt.set_array(spc[pair[1]].T.ravel())
673 673 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
674 674
675 675 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
676 676 coh = numpy.abs(out)
677 677 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
678 678
679 679 ax = self.axes[4 * n + 2]
680 680 if ax.firsttime:
681 681 ax.plt = ax.pcolormesh(x, y, coh.T,
682 682 vmin=0,
683 683 vmax=1,
684 684 cmap=plt.get_cmap(self.colormap_coh)
685 685 )
686 686 else:
687 687 ax.plt.set_array(coh.T.ravel())
688 688 self.titles.append(
689 689 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
690 690
691 691 ax = self.axes[4 * n + 3]
692 692 if ax.firsttime:
693 693 ax.plt = ax.pcolormesh(x, y, phase.T,
694 694 vmin=-180,
695 695 vmax=180,
696 696 cmap=plt.get_cmap(self.colormap_phase)
697 697 )
698 698 else:
699 699 ax.plt.set_array(phase.T.ravel())
700 700 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
701 701
702 702 self.saveTime = self.max_time
703 703
704 704
705 705 class PlotSpectraMeanData(PlotSpectraData):
706 706 '''
707 707 Plot for Spectra and Mean
708 708 '''
709 709 CODE = 'spc_mean'
710 710 colormap = 'jro'
711 711
712 712
713 713 class PlotRTIData(PlotData):
714 714 '''
715 715 Plot for RTI data
716 716 '''
717 717
718 718 CODE = 'rti'
719 719 colormap = 'jro'
720 720
721 721 def setup(self):
722 722 self.xaxis = 'time'
723 723 self.ncols = 1
724 724 self.nrows = len(self.data.channels)
725 725 self.nplots = len(self.data.channels)
726 726 self.ylabel = 'Range [km]'
727 727 self.cb_label = 'dB'
728 728 self.titles = ['{} Channel {}'.format(
729 729 self.CODE.upper(), x) for x in range(self.nrows)]
730 730
731 731 def plot(self):
732 732 self.x = self.times
733 733 self.y = self.data.heights
734 734 self.z = self.data[self.CODE]
735 735 self.z = numpy.ma.masked_invalid(self.z)
736 736
737 737 if self.decimation is None:
738 738 x, y, z = self.fill_gaps(self.x, self.y, self.z)
739 739 else:
740 740 x, y, z = self.fill_gaps(*self.decimate())
741 741
742 742 for n, ax in enumerate(self.axes):
743 743 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
744 744 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
745 745 if ax.firsttime:
746 746 ax.plt = ax.pcolormesh(x, y, z[n].T,
747 747 vmin=self.zmin,
748 748 vmax=self.zmax,
749 749 cmap=plt.get_cmap(self.colormap)
750 750 )
751 751 if self.showprofile:
752 752 ax.plot_profile = self.pf_axes[n].plot(
753 753 self.data['rti'][n][-1], self.y)[0]
754 754 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y,
755 755 color="k", linestyle="dashed", lw=1)[0]
756 756 else:
757 757 ax.collections.remove(ax.collections[0])
758 758 ax.plt = ax.pcolormesh(x, y, z[n].T,
759 759 vmin=self.zmin,
760 760 vmax=self.zmax,
761 761 cmap=plt.get_cmap(self.colormap)
762 762 )
763 763 if self.showprofile:
764 764 ax.plot_profile.set_data(self.data['rti'][n][-1], self.y)
765 765 ax.plot_noise.set_data(numpy.repeat(
766 766 self.data['noise'][n][-1], len(self.y)), self.y)
767 767
768 768 self.saveTime = self.min_time
769 769
770 770
771 771 class PlotCOHData(PlotRTIData):
772 772 '''
773 773 Plot for Coherence data
774 774 '''
775 775
776 776 CODE = 'coh'
777 777
778 778 def setup(self):
779 779 self.xaxis = 'time'
780 780 self.ncols = 1
781 781 self.nrows = len(self.data.pairs)
782 782 self.nplots = len(self.data.pairs)
783 783 self.ylabel = 'Range [km]'
784 784 if self.CODE == 'coh':
785 785 self.cb_label = ''
786 786 self.titles = [
787 787 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
788 788 else:
789 789 self.cb_label = 'Degrees'
790 790 self.titles = [
791 791 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
792 792
793 793
794 794 class PlotPHASEData(PlotCOHData):
795 795 '''
796 796 Plot for Phase map data
797 797 '''
798 798
799 799 CODE = 'phase'
800 800 colormap = 'seismic'
801 801
802 802
803 803 class PlotNoiseData(PlotData):
804 804 '''
805 805 Plot for noise
806 806 '''
807 807
808 808 CODE = 'noise'
809 809
810 810 def setup(self):
811 811 self.xaxis = 'time'
812 812 self.ncols = 1
813 813 self.nrows = 1
814 814 self.nplots = 1
815 815 self.ylabel = 'Intensity [dB]'
816 816 self.titles = ['Noise']
817 817 self.colorbar = False
818 818
819 819 def plot(self):
820 820
821 821 x = self.times
822 822 xmin = self.min_time
823 823 xmax = xmin + self.xrange * 60 * 60
824 824 Y = self.data[self.CODE]
825 825
826 826 if self.axes[0].firsttime:
827 827 for ch in self.data.channels:
828 828 y = Y[ch]
829 829 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
830 830 plt.legend()
831 831 else:
832 832 for ch in self.data.channels:
833 833 y = Y[ch]
834 834 self.axes[0].lines[ch].set_data(x, y)
835 835
836 836 self.ymin = numpy.nanmin(Y) - 5
837 837 self.ymax = numpy.nanmax(Y) + 5
838 838 self.saveTime = self.min_time
839 839
840 840
841 841 class PlotSNRData(PlotRTIData):
842 842 '''
843 843 Plot for SNR Data
844 844 '''
845 845
846 846 CODE = 'snr'
847 847 colormap = 'jet'
848 848
849 849
850 850 class PlotDOPData(PlotRTIData):
851 851 '''
852 852 Plot for DOPPLER Data
853 853 '''
854 854
855 855 CODE = 'dop'
856 856 colormap = 'jet'
857 857
858 858
859 859 class PlotSkyMapData(PlotData):
860 860 '''
861 861 Plot for meteors detection data
862 862 '''
863 863
864 864 CODE = 'param'
865 865
866 866 def setup(self):
867 867
868 868 self.ncols = 1
869 869 self.nrows = 1
870 870 self.width = 7.2
871 871 self.height = 7.2
872 872 self.nplots = 1
873 873 self.xlabel = 'Zonal Zenith Angle (deg)'
874 874 self.ylabel = 'Meridional Zenith Angle (deg)'
875 875 self.polar = True
876 876 self.ymin = -180
877 877 self.ymax = 180
878 878 self.colorbar = False
879 879
880 880 def plot(self):
881 881
882 882 arrayParameters = numpy.concatenate(self.data['param'])
883 883 error = arrayParameters[:, -1]
884 884 indValid = numpy.where(error == 0)[0]
885 885 finalMeteor = arrayParameters[indValid, :]
886 886 finalAzimuth = finalMeteor[:, 3]
887 887 finalZenith = finalMeteor[:, 4]
888 888
889 889 x = finalAzimuth * numpy.pi / 180
890 890 y = finalZenith
891 891
892 892 ax = self.axes[0]
893 893
894 894 if ax.firsttime:
895 895 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
896 896 else:
897 897 ax.plot.set_data(x, y)
898 898
899 899 dt1 = self.getDateTime(self.min_time).strftime('%y/%m/%d %H:%M:%S')
900 900 dt2 = self.getDateTime(self.max_time).strftime('%y/%m/%d %H:%M:%S')
901 901 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
902 902 dt2,
903 903 len(x))
904 904 self.titles[0] = title
905 905 self.saveTime = self.max_time
906 906
907 907
908 908 class PlotParamData(PlotRTIData):
909 909 '''
910 910 Plot for data_param object
911 911 '''
912 912
913 913 CODE = 'param'
914 914 colormap = 'seismic'
915 915
916 916 def setup(self):
917 917 self.xaxis = 'time'
918 918 self.ncols = 1
919 919 self.nrows = self.data.shape(self.CODE)[0]
920 920 self.nplots = self.nrows
921 921 if self.showSNR:
922 922 self.nrows += 1
923 923 self.nplots += 1
924 924
925 925 self.ylabel = 'Height [km]'
926 926 if not self.titles:
927 927 self.titles = self.data.parameters \
928 928 if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
929 929 if self.showSNR:
930 930 self.titles.append('SNR')
931 931
932 932 def plot(self):
933 933 self.data.normalize_heights()
934 934 self.x = self.times
935 935 self.y = self.data.heights
936 936 if self.showSNR:
937 937 self.z = numpy.concatenate(
938 938 (self.data[self.CODE], self.data['snr'])
939 939 )
940 940 else:
941 941 self.z = self.data[self.CODE]
942 942
943 943 self.z = numpy.ma.masked_invalid(self.z)
944 944
945 945 if self.decimation is None:
946 946 x, y, z = self.fill_gaps(self.x, self.y, self.z)
947 947 else:
948 948 x, y, z = self.fill_gaps(*self.decimate())
949 949
950 950 for n, ax in enumerate(self.axes):
951 951
952 952 self.zmax = self.zmax if self.zmax is not None else numpy.max(
953 953 self.z[n])
954 954 self.zmin = self.zmin if self.zmin is not None else numpy.min(
955 955 self.z[n])
956 956
957 957 if ax.firsttime:
958 958 if self.zlimits is not None:
959 959 self.zmin, self.zmax = self.zlimits[n]
960 960
961 961 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
962 962 vmin=self.zmin,
963 963 vmax=self.zmax,
964 964 cmap=self.cmaps[n]
965 965 )
966 966 else:
967 967 if self.zlimits is not None:
968 968 self.zmin, self.zmax = self.zlimits[n]
969 969 ax.collections.remove(ax.collections[0])
970 970 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
971 971 vmin=self.zmin,
972 972 vmax=self.zmax,
973 973 cmap=self.cmaps[n]
974 974 )
975 975
976 976 self.saveTime = self.min_time
977 977
978 978
979 979 class PlotOutputData(PlotParamData):
980 980 '''
981 981 Plot data_output object
982 982 '''
983 983
984 984 CODE = 'output'
985 985 colormap = 'seismic'
986 986
987 987
988 988 class PlotPolarMapData(PlotData):
989 989 '''
990 990 Plot for meteors detection data
991 991 '''
992 992
993 993 CODE = 'param'
994 994 colormap = 'seismic'
995 995
996 996 def setup(self):
997 997 self.ncols = 1
998 998 self.nrows = 1
999 999 self.width = 9
1000 1000 self.height = 8
1001 1001 if self.channels is not None:
1002 1002 self.nplots = len(self.channels)
1003 1003 self.nrows = len(self.channels)
1004 1004 else:
1005 1005 self.nplots = self.data.shape(self.CODE)[0]
1006 1006 self.nrows = self.nplots
1007 1007 self.channels = range(self.nplots)
1008 1008 if self.data.meta['mode'] == 'E':
1009 1009 self.xlabel = 'Zonal Distance (km)'
1010 1010 self.ylabel = 'Meridional Distance (km)'
1011 1011 else:
1012 1012 self.xlabel = 'Range (km)'
1013 1013 self.ylabel = 'Height (km)'
1014 1014 self.bgcolor = 'white'
1015 1015 self.cb_labels = self.data.meta['units']
1016 1016 # self.polar = True
1017 1017
1018 1018 def plot(self):
1019 1019
1020 1020 for n, ax in enumerate(self.axes):
1021 1021 data = self.data['param'][self.channels[n]]
1022 1022
1023 1023 zeniths = numpy.linspace(0, self.data.meta['max_range'], data.shape[1])
1024 1024 if self.data.meta['mode'] == 'E':
1025 1025 azimuths = -numpy.radians(self.data.heights)+numpy.pi/2
1026 1026 r, theta = numpy.meshgrid(zeniths, azimuths)
1027 1027 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
1028 1028 else:
1029 1029 azimuths = numpy.radians(self.data.heights)
1030 1030 r, theta = numpy.meshgrid(zeniths, azimuths)
1031 1031 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
1032 1032 self.y = zeniths
1033 1033
1034 1034 if ax.firsttime:
1035 1035 if self.zlimits is not None:
1036 1036 self.zmin, self.zmax = self.zlimits[n]
1037 1037 ax.plt = ax.pcolormesh(#r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
1038 1038 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
1039 1039 vmin=self.zmin,
1040 1040 vmax=self.zmax,
1041 1041 cmap=self.cmaps[n])
1042 1042 else:
1043 1043 if self.zlimits is not None:
1044 1044 self.zmin, self.zmax = self.zlimits[n]
1045 1045 ax.collections.remove(ax.collections[0])
1046 1046 ax.plt = ax.pcolormesh(# r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
1047 1047 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
1048 1048 vmin=self.zmin,
1049 1049 vmax=self.zmax,
1050 1050 cmap=self.cmaps[n])
1051 1051
1052 1052 if self.data.meta['mode'] == 'A':
1053 1053 continue
1054 1054 '''
1055 1055 f = open('/data/workspace/schain_scripts/map_lima.csv')
1056 1056
1057 1057 lat1 = -11.96
1058 1058 lon1 = -76.54
1059 1059
1060 1060 for line in f:
1061 1061 label, x, y = [s.strip() for s in line.split(',') if s]
1062 1062 lat2 = float(y)
1063 1063 lon2 = float(x)
1064 1064
1065 1065 dx = (lon2-lon1)*40000*numpy.cos((lat1+lat2)*numpy.pi/360)/360
1066 1066 dy = (lat1-lat2)*40000/360
1067 1067 print label, dx, dy
1068 1068 if label == 'map':
1069 1069 print 'SDHSDHSDHSGHSDFHSDF'
1070 1070 ax.plot([dx], [dy],'--k')
1071 1071 else:
1072 1072 ax.plot([dx], [dy],'.b', ms=2)
1073 1073 '''
1074 1074 if self.data.meta['mode'] == 'E':
1075 1075 title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
1076 1076 label = 'E{:d}'.format(int(self.data.meta['elevation']))
1077 1077 else:
1078 1078 title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
1079 1079 label = 'A{:d}'.format(int(self.data.meta['azimuth']))
1080 1080
1081 1081 self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
1082 1082 self.titles = ['{} {}'.format(self.data.parameters[x], title) for x in self.channels]
1083 1083 self.saveTime = self.max_time
1084 1084
1085 1085
Show file before | Show file after | Hide comments
@@ -1,361 +1,345
1 1 '''
2 Created on Dec 27, 2017
2 Created on Jan 15, 2018
3 3
4 4 @author: Juan C. Espinoza
5 5 '''
6 6
7 7 import os
8 8 import sys
9 9 import time
10 import json
11 10 import glob
12 11 import datetime
13 12 import tarfile
14 13
15 14 import numpy
16 15 from netCDF4 import Dataset
17 16
18 17 from utils import folder_in_range
19 18
20 19 from schainpy.model.io.jroIO_base import JRODataReader
21 20 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
22 21 from schainpy.model.data.jrodata import Parameters
23 22 from schainpy.utils import log
24 23
25 24 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
26 25
27 def load_json(obj):
28 '''
29 Parse json as string instead of unicode
30 '''
31
32 if isinstance(obj, str):
33 iterable = json.loads(obj)
34 else:
35 iterable = obj
36
37 if isinstance(iterable, dict):
38 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
39 for k, v in iterable.items()}
40 elif isinstance(iterable, (list, tuple)):
41 return [str(v) if isinstance(v, unicode) else v for v in iterable]
42
43 return iterable
44
45 26
46 27 class PXReader(JRODataReader, ProcessingUnit):
47 28
48 29 def __init__(self, **kwargs):
49 30
50 31 ProcessingUnit.__init__(self, **kwargs)
51 32
52 33 self.dataOut = Parameters()
53 34 self.counter_records = 0
54 35 self.nrecords = None
55 36 self.flagNoMoreFiles = 0
56 37 self.isConfig = False
57 38 self.filename = None
58 39 self.intervals = set()
59 40 self.ext = ('.nc', '.tgz')
60 41 self.online_mode = False
61 42
62 43 def setup(self,
63 44 path=None,
64 45 startDate=None,
65 46 endDate=None,
66 47 format=None,
67 48 startTime=datetime.time(0, 0, 0),
68 49 endTime=datetime.time(23, 59, 59),
69 50 walk=False,
70 51 **kwargs):
71 52
72 53 self.path = path
73 54 self.startDate = startDate
74 55 self.endDate = endDate
75 56 self.startTime = startTime
76 57 self.endTime = endTime
77 58 self.datatime = datetime.datetime(1900,1,1)
78 59 self.walk = walk
79 self.nTries = kwargs.get('nTries', 3)
60 self.nTries = kwargs.get('nTries', 10)
80 61 self.online = kwargs.get('online', False)
81 self.delay = kwargs.get('delay', 30)
62 self.delay = kwargs.get('delay', 60)
82 63 self.ele = kwargs.get('ext', '')
83 64
84 65 if self.path is None:
85 66 raise ValueError, 'The path is not valid'
86 67
87 68 self.search_files(path, startDate, endDate, startTime, endTime, walk)
88 69 self.cursor = 0
89 70 self.counter_records = 0
90 71
91 72 if not self.files:
92 73 raise Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path)
93 74
94 75 def search_files(self, path, startDate, endDate, startTime, endTime, walk):
95 76 '''
96 77 Searching for NCDF files in path
97 78 Creating a list of files to procces included in [startDate,endDate]
98 79
99 80 Input:
100 81 path - Path to find files
101 82 '''
102 83
103 84 log.log('Searching files {} in {} '.format(self.ext, path), 'PXReader')
104 85 if walk:
105 86 paths = [os.path.join(path, p) for p in os.listdir(path) if os.path.isdir(os.path.join(path, p))]
106 87 paths.sort()
107 88 else:
108 89 paths = [path]
109 90
110 91 fileList0 = []
111 92
112 93 for subpath in paths:
113 94 if not folder_in_range(subpath.split('/')[-1], startDate, endDate, '%Y%m%d'):
114 95 continue
115 96 fileList0 += [os.path.join(subpath, s) for s in glob.glob1(subpath, '*') if os.path.splitext(s)[-1] in self.ext and '{}'.format(self.ele) in s]
116 97
117 98 fileList0.sort()
118 99 if self.online:
119 100 fileList0 = fileList0[-1:]
120 101
121 102 self.files = {}
122 103
123 104 startDate = startDate - datetime.timedelta(1)
124 105 endDate = endDate + datetime.timedelta(1)
125 106
126 107 for fullname in fileList0:
127 108 thisFile = fullname.split('/')[-1]
128 109 year = thisFile[3:7]
129 110 if not year.isdigit():
130 111 continue
131 112
132 113 month = thisFile[7:9]
133 114 if not month.isdigit():
134 115 continue
135 116
136 117 day = thisFile[9:11]
137 118 if not day.isdigit():
138 119 continue
139 120
140 121 year, month, day = int(year), int(month), int(day)
141 122 dateFile = datetime.date(year, month, day)
142 123 timeFile = datetime.time(int(thisFile[12:14]), int(thisFile[14:16]), int(thisFile[16:18]))
143 124
144 125 if (startDate > dateFile) or (endDate < dateFile):
145 126 continue
146 127
147 128 dt = datetime.datetime.combine(dateFile, timeFile)
148 129 if dt not in self.files:
149 130 self.files[dt] = []
150 131 self.files[dt].append(fullname)
151 132
152 133 self.dates = self.files.keys()
153 134 self.dates.sort()
154 135
155 136 return
156 137
157 138 def search_files_online(self):
158 139 '''
159 140 Searching for NCDF files in online mode path
160 141 Creating a list of files to procces included in [startDate,endDate]
161 142
162 143 Input:
163 144 path - Path to find files
164 145 '''
165 146
166 147 self.files = {}
167 148
168 149 for n in range(self.nTries):
169 150
170 151 if self.walk:
171 152 paths = [os.path.join(self.path, p) for p in os.listdir(self.path) if os.path.isdir(os.path.join(self.path, p))]
153 paths.sort()
172 154 path = paths[-1]
173 155 else:
174 paths = self.path
156 path = self.path
175 157
176 new_files = [os.path.join(path, s) for s in glob.glob1(path, '*') if os.path.splitext(s)[-1] in self.ext and '{}'.format(self.ele) in s]
177
158 new_files = [os.path.join(path, s) for s in glob.glob1(path, '*') if os.path.splitext(s)[-1] in self.ext and '{}'.format(self.ele) in s]
178 159 new_files.sort()
179 160
180 161 for fullname in new_files:
181 162 thisFile = fullname.split('/')[-1]
182 163 year = thisFile[3:7]
183 164 if not year.isdigit():
184 165 continue
185 166
186 167 month = thisFile[7:9]
187 168 if not month.isdigit():
188 169 continue
189 170
190 171 day = thisFile[9:11]
191 172 if not day.isdigit():
192 173 continue
193 174
194 175 year, month, day = int(year), int(month), int(day)
195 176 dateFile = datetime.date(year, month, day)
196 177 timeFile = datetime.time(int(thisFile[12:14]), int(thisFile[14:16]), int(thisFile[16:18]))
197 178
198 179 dt = datetime.datetime.combine(dateFile, timeFile)
199 180
200 181 if self.dt >= dt:
201 182 continue
202 183
203 184 if dt not in self.files:
204 185 self.dt = dt
205 186 self.files[dt] = []
206 187
207 188 self.files[dt].append(fullname)
208 189 break
209 190
210 191 if self.files:
211 192 break
212 193 else:
213 194 log.warning('Waiting {} seconds for the next file, try {} ...'.format(self.delay, n + 1), 'PXReader')
214 195 time.sleep(self.delay)
215 196
216 197 if not self.files:
217 198 return 0
218 199
219 200 self.dates = self.files.keys()
220 201 self.dates.sort()
221 202 self.cursor = 0
222 203
223 204 return 1
224 205
225 206 def parseFile(self):
226 207 '''
227 208 '''
228 209
229 210 header = {}
230 211
231 212 for attr in self.fp.ncattrs():
232 213 header[str(attr)] = getattr(self.fp, attr)
233 214
234 215 self.header.append(header)
235 216
236 217 self.data[header['TypeName']] = numpy.array(self.fp.variables[header['TypeName']])
237 218
238 219 def setNextFile(self):
239 220 '''
240 221 Open next files for the current datetime
241 222 '''
242 223
243 224 cursor = self.cursor
244 225 if not self.online_mode:
245 226 if cursor == len(self.dates):
246 227 if self.online:
247 228 cursor = 0
248 229 self.dt = self.dates[cursor]
249 230 self.online_mode = True
250 231 if not self.search_files_online():
251 232 log.success('No more files', 'PXReader')
252 233 return 0
253 234 else:
254 235 log.success('No more files', 'PXReader')
255 236 self.flagNoMoreFiles = 1
256 237 return 0
257 238 else:
258 239 if not self.search_files_online():
259 240 return 0
260 241 cursor = self.cursor
261 242
262 243 self.data = {}
263 244 self.header = []
264 245
265 246 for fullname in self.files[self.dates[cursor]]:
266 247
267 248 log.log('Opening: {}'.format(fullname), 'PXReader')
268 249
269 250 if os.path.splitext(fullname)[-1] == '.tgz':
270 251 tar = tarfile.open(fullname, 'r:gz')
271 252 tar.extractall('/tmp')
272 253 files = [os.path.join('/tmp', member.name) for member in tar.getmembers()]
273 254 else:
274 255 files = [fullname]
275 256
276 257 for filename in files:
277 258 if self.filename is not None:
278 259 self.fp.close()
279 260
280 261 self.filename = filename
281 262 self.filedate = self.dates[cursor]
282 263 self.fp = Dataset(self.filename, 'r')
283 264 self.parseFile()
284 265
285 266 self.counter_records += 1
286 267 self.cursor += 1
287 268 return 1
288 269
289 270 def readNextFile(self):
290 271
291 272 while True:
292 273 self.flagDiscontinuousBlock = 0
293 274 if not self.setNextFile():
294 275 return 0
295 276
296 277 self.datatime = datetime.datetime.utcfromtimestamp(self.header[0]['Time'])
297 278
279 if self.online:
280 break
281
298 282 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
299 283 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
300 284 log.warning(
301 285 'Reading Record No. {}/{} -> {} [Skipping]'.format(
302 286 self.counter_records,
303 287 self.nrecords,
304 288 self.datatime.ctime()),
305 289 'PXReader')
306 290 continue
307 291 break
308 292
309 293 log.log(
310 294 'Reading Record No. {}/{} -> {}'.format(
311 295 self.counter_records,
312 296 self.nrecords,
313 297 self.datatime.ctime()),
314 298 'PXReader')
315 299
316 300 return 1
317 301
318 302
319 303 def set_output(self):
320 304 '''
321 305 Storing data from buffer to dataOut object
322 306 '''
323 307
324 308 self.data['Elevation'] = numpy.array(self.fp.variables['Elevation'])
325 309 self.data['Azimuth'] = numpy.array(self.fp.variables['Azimuth'])
326 310 self.dataOut.range = numpy.array(self.fp.variables['GateWidth'])
327 311 self.dataOut.data = self.data
328 312 self.dataOut.units = [h['Unit-value'] for h in self.header]
329 313 self.dataOut.parameters = [h['TypeName'] for h in self.header]
330 314 self.dataOut.missing = self.header[0]['MissingData']
331 315 self.dataOut.max_range = self.header[0]['MaximumRange-value']
332 316 self.dataOut.elevation = self.header[0]['Elevation']
333 317 self.dataOut.azimuth = self.header[0]['Azimuth']
334 318 self.dataOut.latitude = self.header[0]['Latitude']
335 319 self.dataOut.longitude = self.header[0]['Longitude']
336 320 self.dataOut.utctime = self.header[0]['Time']
337 321 self.dataOut.utctimeInit = self.dataOut.utctime
338 322 self.dataOut.useLocalTime = True
339 323 self.dataOut.flagNoData = False
340 324 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
341 325
342 326 log.log('Parameters found: {}'.format(','.join(self.dataOut.parameters)),
343 327 'PXReader')
344 328
345 329 def getData(self):
346 330 '''
347 331 Storing data from databuffer to dataOut object
348 332 '''
349 333 if self.flagNoMoreFiles:
350 334 self.dataOut.flagNoData = True
351 335 log.error('No file left to process', 'PXReader')
352 336 return 0
353 337
354 338 if not self.readNextFile():
355 339 self.dataOut.flagNoData = True
356 340 return 0
357 341
358 342 self.set_output()
359 343
360 344 return 1
361 345
General Comments 0
You need to be logged in to leave comments. Login now