##// END OF EJS Templates
schain-jc
RSS
MADReader support for HDF5 (mad2 & mad3)
jespinoza -
r1065:9a4424b32cac
parent child
Show More
Side by Side Unified
Context file:
r1065:9a4424b32cac Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -1,7 +1,7
1 1 '''
2 2 Created on Feb 7, 2012
3 3
4 4 @author $Author$
5 5 @version $Id$
6 6 '''
7 __version__ = "2.3"
7 __version__ = '2.3'
Show file before | Show file after | Hide comments
@@ -1,782 +1,783
1 1
2 2 import os
3 3 import time
4 4 import glob
5 5 import datetime
6 6 from multiprocessing import Process
7 7
8 8 import zmq
9 9 import numpy
10 10 import matplotlib
11 11 import matplotlib.pyplot as plt
12 12 from mpl_toolkits.axes_grid1 import make_axes_locatable
13 13 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
14 14
15 15 from schainpy.model.proc.jroproc_base import Operation
16 16 from schainpy.utils import log
17 17
18 18 func = lambda x, pos: ('%s') %(datetime.datetime.fromtimestamp(x).strftime('%H:%M'))
19 19
20 20 d1970 = datetime.datetime(1970, 1, 1)
21 21
22 22
23 23 class PlotData(Operation, Process):
24 24 '''
25 25 Base class for Schain plotting operations
26 26 '''
27 27
28 28 CODE = 'Figure'
29 29 colormap = 'jro'
30 30 bgcolor = 'white'
31 31 CONFLATE = False
32 32 __MAXNUMX = 80
33 33 __missing = 1E30
34 34
35 35 def __init__(self, **kwargs):
36 36
37 37 Operation.__init__(self, plot=True, **kwargs)
38 38 Process.__init__(self)
39 39 self.kwargs['code'] = self.CODE
40 40 self.mp = False
41 41 self.data = None
42 42 self.isConfig = False
43 43 self.figures = []
44 44 self.axes = []
45 45 self.cb_axes = []
46 46 self.localtime = kwargs.pop('localtime', True)
47 47 self.show = kwargs.get('show', True)
48 48 self.save = kwargs.get('save', False)
49 49 self.colormap = kwargs.get('colormap', self.colormap)
50 50 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
51 51 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
52 52 self.colormaps = kwargs.get('colormaps', None)
53 53 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
54 54 self.showprofile = kwargs.get('showprofile', False)
55 55 self.title = kwargs.get('wintitle', self.CODE.upper())
56 56 self.cb_label = kwargs.get('cb_label', None)
57 57 self.cb_labels = kwargs.get('cb_labels', None)
58 58 self.xaxis = kwargs.get('xaxis', 'frequency')
59 59 self.zmin = kwargs.get('zmin', None)
60 60 self.zmax = kwargs.get('zmax', None)
61 61 self.zlimits = kwargs.get('zlimits', None)
62 62 self.xmin = kwargs.get('xmin', None)
63 63 if self.xmin is not None:
64 64 self.xmin += 5
65 65 self.xmax = kwargs.get('xmax', None)
66 66 self.xrange = kwargs.get('xrange', 24)
67 67 self.ymin = kwargs.get('ymin', None)
68 68 self.ymax = kwargs.get('ymax', None)
69 69 self.xlabel = kwargs.get('xlabel', None)
70 70 self.__MAXNUMY = kwargs.get('decimation', 100)
71 71 self.showSNR = kwargs.get('showSNR', False)
72 72 self.oneFigure = kwargs.get('oneFigure', True)
73 73 self.width = kwargs.get('width', None)
74 74 self.height = kwargs.get('height', None)
75 75 self.colorbar = kwargs.get('colorbar', True)
76 76 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
77 77 self.titles = ['' for __ in range(16)]
78 78
79 79 def __setup(self):
80 80 '''
81 81 Common setup for all figures, here figures and axes are created
82 82 '''
83 83
84 84 self.setup()
85 85
86 86 if self.width is None:
87 87 self.width = 8
88 88
89 89 self.figures = []
90 90 self.axes = []
91 91 self.cb_axes = []
92 92 self.pf_axes = []
93 93 self.cmaps = []
94 94
95 95 size = '15%' if self.ncols==1 else '30%'
96 96 pad = '4%' if self.ncols==1 else '8%'
97 97
98 98 if self.oneFigure:
99 99 if self.height is None:
100 100 self.height = 1.4*self.nrows + 1
101 101 fig = plt.figure(figsize=(self.width, self.height),
102 102 edgecolor='k',
103 103 facecolor='w')
104 104 self.figures.append(fig)
105 105 for n in range(self.nplots):
106 106 ax = fig.add_subplot(self.nrows, self.ncols, n+1)
107 107 ax.tick_params(labelsize=8)
108 108 ax.firsttime = True
109 109 self.axes.append(ax)
110 110 if self.showprofile:
111 111 cax = self.__add_axes(ax, size=size, pad=pad)
112 112 cax.tick_params(labelsize=8)
113 113 self.pf_axes.append(cax)
114 114 else:
115 115 if self.height is None:
116 116 self.height = 3
117 117 for n in range(self.nplots):
118 118 fig = plt.figure(figsize=(self.width, self.height),
119 119 edgecolor='k',
120 120 facecolor='w')
121 121 ax = fig.add_subplot(1, 1, 1)
122 122 ax.tick_params(labelsize=8)
123 123 ax.firsttime = True
124 124 self.figures.append(fig)
125 125 self.axes.append(ax)
126 126 if self.showprofile:
127 127 cax = self.__add_axes(ax, size=size, pad=pad)
128 128 cax.tick_params(labelsize=8)
129 129 self.pf_axes.append(cax)
130 130
131 131 for n in range(self.nrows):
132 132 if self.colormaps is not None:
133 133 cmap = plt.get_cmap(self.colormaps[n])
134 134 else:
135 135 cmap = plt.get_cmap(self.colormap)
136 136 cmap.set_bad(self.bgcolor, 1.)
137 137 self.cmaps.append(cmap)
138 138
139 139 def __add_axes(self, ax, size='30%', pad='8%'):
140 140 '''
141 141 Add new axes to the given figure
142 142 '''
143 143 divider = make_axes_locatable(ax)
144 144 nax = divider.new_horizontal(size=size, pad=pad)
145 145 ax.figure.add_axes(nax)
146 146 return nax
147 147
148 self.setup()
148 149
149 150 def setup(self):
150 151 '''
151 152 This method should be implemented in the child class, the following
152 153 attributes should be set:
153 154
154 155 self.nrows: number of rows
155 156 self.ncols: number of cols
156 157 self.nplots: number of plots (channels or pairs)
157 158 self.ylabel: label for Y axes
158 159 self.titles: list of axes title
159 160
160 161 '''
161 162 raise(NotImplementedError, 'Implement this method in child class')
162 163
163 164 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
164 165 '''
165 166 Create a masked array for missing data
166 167 '''
167 168 if x_buffer.shape[0] < 2:
168 169 return x_buffer, y_buffer, z_buffer
169 170
170 171 deltas = x_buffer[1:] - x_buffer[0:-1]
171 172 x_median = numpy.median(deltas)
172 173
173 174 index = numpy.where(deltas > 5*x_median)
174 175
175 176 if len(index[0]) != 0:
176 177 z_buffer[::, index[0], ::] = self.__missing
177 178 z_buffer = numpy.ma.masked_inside(z_buffer,
178 179 0.99*self.__missing,
179 180 1.01*self.__missing)
180 181
181 182 return x_buffer, y_buffer, z_buffer
182 183
183 184 def decimate(self):
184 185
185 186 # dx = int(len(self.x)/self.__MAXNUMX) + 1
186 187 dy = int(len(self.y)/self.__MAXNUMY) + 1
187 188
188 189 # x = self.x[::dx]
189 190 x = self.x
190 191 y = self.y[::dy]
191 192 z = self.z[::, ::, ::dy]
192 193
193 194 return x, y, z
194 195
195 196 def format(self):
196 197 '''
197 198 Set min and max values, labels, ticks and titles
198 199 '''
199 200
200 201 if self.xmin is None:
201 202 xmin = self.min_time
202 203 else:
203 204 if self.xaxis is 'time':
204 205 dt = datetime.datetime.fromtimestamp(self.min_time)
205 206 xmin = (datetime.datetime.combine(dt.date(),
206 207 datetime.time(int(self.xmin), 0, 0))-d1970).total_seconds()
207 208 else:
208 209 xmin = self.xmin
209 210
210 211 if self.xmax is None:
211 212 xmax = xmin+self.xrange*60*60
212 213 else:
213 214 if self.xaxis is 'time':
214 215 dt = datetime.datetime.fromtimestamp(self.min_time)
215 216 xmax = (datetime.datetime.combine(dt.date(),
216 217 datetime.time(int(self.xmax), 0, 0))-d1970).total_seconds()
217 218 else:
218 219 xmax = self.xmax
219 220
220 221 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
221 222 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
222 223
223 224 ystep = 200 if ymax>= 800 else 100 if ymax>=400 else 50 if ymax>=200 else 20
224 225
225 226 for n, ax in enumerate(self.axes):
226 227 if ax.firsttime:
227 228 ax.set_facecolor(self.bgcolor)
228 229 ax.yaxis.set_major_locator(MultipleLocator(ystep))
229 230 if self.xaxis is 'time':
230 231 ax.xaxis.set_major_formatter(FuncFormatter(func))
231 232 ax.xaxis.set_major_locator(LinearLocator(9))
232 233 if self.xlabel is not None:
233 234 ax.set_xlabel(self.xlabel)
234 235 ax.set_ylabel(self.ylabel)
235 236 ax.firsttime = False
236 237 if self.showprofile:
237 238 self.pf_axes[n].set_ylim(ymin, ymax)
238 239 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
239 240 self.pf_axes[n].set_xlabel('dB')
240 241 self.pf_axes[n].grid(b=True, axis='x')
241 242 [tick.set_visible(False) for tick in self.pf_axes[n].get_yticklabels()]
242 243 if self.colorbar:
243 244 cb = plt.colorbar(ax.plt, ax=ax, pad=0.02)
244 245 cb.ax.tick_params(labelsize=8)
245 246 if self.cb_label:
246 247 cb.set_label(self.cb_label, size=8)
247 248 elif self.cb_labels:
248 249 cb.set_label(self.cb_labels[n], size=8)
249 250
250 251 ax.set_title('{} - {} UTC'.format(
251 252 self.titles[n],
252 253 datetime.datetime.fromtimestamp(self.max_time).strftime('%H:%M:%S')),
253 254 size=8)
254 255 ax.set_xlim(xmin, xmax)
255 256 ax.set_ylim(ymin, ymax)
256 257
257 258
258 259 def __plot(self):
259 260 '''
260 261 '''
261 262 log.success('Plotting', self.name)
262 263
263 264 self.plot()
264 265 self.format()
265 266
266 267 for n, fig in enumerate(self.figures):
267 268 if self.nrows == 0 or self.nplots == 0:
268 269 log.warning('No data', self.name)
269 270 continue
270 271 if self.show:
271 272 fig.show()
272 273
273 274 fig.tight_layout()
274 275 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
275 276 datetime.datetime.fromtimestamp(self.max_time).strftime('%Y/%m/%d')))
276 277 # fig.canvas.draw()
277 278
278 279 if self.save and self.data.ended:
279 280 channels = range(self.nrows)
280 281 if self.oneFigure:
281 282 label = ''
282 283 else:
283 284 label = '_{}'.format(channels[n])
284 285 figname = os.path.join(
285 286 self.save,
286 287 '{}{}_{}.png'.format(
287 288 self.CODE,
288 289 label,
289 290 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')
290 291 )
291 292 )
292 293 print 'Saving figure: {}'.format(figname)
293 294 fig.savefig(figname)
294 295
295 296 def plot(self):
296 297 '''
297 298 '''
298 299 raise(NotImplementedError, 'Implement this method in child class')
299 300
300 301 def run(self):
301 302
302 303 log.success('Starting', self.name)
303 304
304 305 context = zmq.Context()
305 306 receiver = context.socket(zmq.SUB)
306 307 receiver.setsockopt(zmq.SUBSCRIBE, '')
307 308 receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
308 309
309 310 if 'server' in self.kwargs['parent']:
310 311 receiver.connect('ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
311 312 else:
312 313 receiver.connect("ipc:///tmp/zmq.plots")
313 314
314 315 while True:
315 316 try:
316 317 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
317 318
318 319 self.min_time = self.data.times[0]
319 320 self.max_time = self.data.times[-1]
320 321
321 322 if self.isConfig is False:
322 323 self.__setup()
323 324 self.isConfig = True
324 325
325 326 self.__plot()
326 327
327 328 except zmq.Again as e:
328 329 log.log('Waiting for data...')
329 330 if self.data:
330 331 plt.pause(self.data.throttle)
331 332 else:
332 333 time.sleep(2)
333 334
334 335 def close(self):
335 336 if self.data:
336 337 self.__plot()
337 338
338
339 339 class PlotSpectraData(PlotData):
340 340 '''
341 341 Plot for Spectra data
342 342 '''
343 343
344 344 CODE = 'spc'
345 345 colormap = 'jro'
346 346
347 347 def setup(self):
348 348 self.nplots = len(self.data.channels)
349 349 self.ncols = int(numpy.sqrt(self.nplots)+ 0.9)
350 350 self.nrows = int((1.0*self.nplots/self.ncols) + 0.9)
351 351 self.width = 3.4*self.ncols
352 352 self.height = 3*self.nrows
353 353 self.cb_label = 'dB'
354 354 if self.showprofile:
355 355 self.width += 0.8*self.ncols
356 356
357 357 self.ylabel = 'Range [Km]'
358 358
359 359 def plot(self):
360 360 if self.xaxis == "frequency":
361 361 x = self.data.xrange[0]
362 362 self.xlabel = "Frequency (kHz)"
363 363 elif self.xaxis == "time":
364 364 x = self.data.xrange[1]
365 365 self.xlabel = "Time (ms)"
366 366 else:
367 367 x = self.data.xrange[2]
368 368 self.xlabel = "Velocity (m/s)"
369 369
370 370 if self.CODE == 'spc_mean':
371 371 x = self.data.xrange[2]
372 372 self.xlabel = "Velocity (m/s)"
373 373
374 374 self.titles = []
375 375
376 376 y = self.data.heights
377 377 self.y = y
378 378 z = self.data['spc']
379 379
380 380 for n, ax in enumerate(self.axes):
381 381 noise = self.data['noise'][n][-1]
382 382 if self.CODE == 'spc_mean':
383 383 mean = self.data['mean'][n][-1]
384 384 if ax.firsttime:
385 385 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
386 386 self.xmin = self.xmin if self.xmin else -self.xmax
387 387 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
388 388 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
389 389 ax.plt = ax.pcolormesh(x, y, z[n].T,
390 390 vmin=self.zmin,
391 391 vmax=self.zmax,
392 392 cmap=plt.get_cmap(self.colormap)
393 393 )
394 394
395 395 if self.showprofile:
396 396 ax.plt_profile= self.pf_axes[n].plot(self.data['rti'][n][-1], y)[0]
397 397 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
398 398 color="k", linestyle="dashed", lw=1)[0]
399 399 if self.CODE == 'spc_mean':
400 400 ax.plt_mean = ax.plot(mean, y, color='k')[0]
401 401 else:
402 402 ax.plt.set_array(z[n].T.ravel())
403 403 if self.showprofile:
404 404 ax.plt_profile.set_data(self.data['rti'][n][-1], y)
405 405 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
406 406 if self.CODE == 'spc_mean':
407 407 ax.plt_mean.set_data(mean, y)
408 408
409 409 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
410 410 self.saveTime = self.max_time
411 411
412 412
413 413 class PlotCrossSpectraData(PlotData):
414 414
415 415 CODE = 'cspc'
416 416 zmin_coh = None
417 417 zmax_coh = None
418 418 zmin_phase = None
419 419 zmax_phase = None
420 420
421 421 def setup(self):
422 422
423 423 self.ncols = 4
424 424 self.nrows = len(self.data.pairs)
425 425 self.nplots = self.nrows*4
426 426 self.width = 3.4*self.ncols
427 427 self.height = 3*self.nrows
428 428 self.ylabel = 'Range [Km]'
429 429 self.showprofile = False
430 430
431 431 def plot(self):
432 432
433 433 if self.xaxis == "frequency":
434 434 x = self.data.xrange[0]
435 435 self.xlabel = "Frequency (kHz)"
436 436 elif self.xaxis == "time":
437 437 x = self.data.xrange[1]
438 438 self.xlabel = "Time (ms)"
439 439 else:
440 440 x = self.data.xrange[2]
441 441 self.xlabel = "Velocity (m/s)"
442 442
443 443 self.titles = []
444 444
445 445 y = self.data.heights
446 446 self.y = y
447 447 spc = self.data['spc']
448 448 cspc = self.data['cspc']
449 449
450 450 for n in range(self.nrows):
451 451 noise = self.data['noise'][n][-1]
452 452 pair = self.data.pairs[n]
453 453 ax = self.axes[4*n]
454 454 ax3 = self.axes[4*n+3]
455 455 if ax.firsttime:
456 456 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
457 457 self.xmin = self.xmin if self.xmin else -self.xmax
458 458 self.zmin = self.zmin if self.zmin else numpy.nanmin(spc)
459 459 self.zmax = self.zmax if self.zmax else numpy.nanmax(spc)
460 460 ax.plt = ax.pcolormesh(x, y, spc[pair[0]].T,
461 461 vmin=self.zmin,
462 462 vmax=self.zmax,
463 463 cmap=plt.get_cmap(self.colormap)
464 464 )
465 465 else:
466 466 ax.plt.set_array(spc[pair[0]].T.ravel())
467 467 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
468 468
469 469 ax = self.axes[4*n+1]
470 470 if ax.firsttime:
471 471 ax.plt = ax.pcolormesh(x, y, spc[pair[1]].T,
472 472 vmin=self.zmin,
473 473 vmax=self.zmax,
474 474 cmap=plt.get_cmap(self.colormap)
475 475 )
476 476 else:
477 477 ax.plt.set_array(spc[pair[1]].T.ravel())
478 478 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
479 479
480 480 out = cspc[n]/numpy.sqrt(spc[pair[0]]*spc[pair[1]])
481 481 coh = numpy.abs(out)
482 482 phase = numpy.arctan2(out.imag, out.real)*180/numpy.pi
483 483
484 484 ax = self.axes[4*n+2]
485 485 if ax.firsttime:
486 486 ax.plt = ax.pcolormesh(x, y, coh.T,
487 487 vmin=0,
488 488 vmax=1,
489 489 cmap=plt.get_cmap(self.colormap_coh)
490 490 )
491 491 else:
492 492 ax.plt.set_array(coh.T.ravel())
493 493 self.titles.append('Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
494 494
495 495 ax = self.axes[4*n+3]
496 496 if ax.firsttime:
497 497 ax.plt = ax.pcolormesh(x, y, phase.T,
498 498 vmin=-180,
499 499 vmax=180,
500 500 cmap=plt.get_cmap(self.colormap_phase)
501 501 )
502 502 else:
503 503 ax.plt.set_array(phase.T.ravel())
504 504 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
505 505
506 506 self.saveTime = self.max_time
507 507
508 508
509 509 class PlotSpectraMeanData(PlotSpectraData):
510 510 '''
511 511 Plot for Spectra and Mean
512 512 '''
513 513 CODE = 'spc_mean'
514 514 colormap = 'jro'
515 515
516 516
517 517 class PlotRTIData(PlotData):
518 518 '''
519 519 Plot for RTI data
520 520 '''
521 521
522 522 CODE = 'rti'
523 523 colormap = 'jro'
524 524
525 525 def setup(self):
526 526 self.xaxis = 'time'
527 527 self.ncols = 1
528 528 self.nrows = len(self.data.channels)
529 529 self.nplots = len(self.data.channels)
530 530 self.ylabel = 'Range [Km]'
531 531 self.cb_label = 'dB'
532 532 self.titles = ['{} Channel {}'.format(self.CODE.upper(), x) for x in range(self.nrows)]
533 533
534 534 def plot(self):
535 535 self.x = self.data.times
536 536 self.y = self.data.heights
537 537 self.z = self.data[self.CODE]
538 538 self.z = numpy.ma.masked_invalid(self.z)
539 539
540 540 for n, ax in enumerate(self.axes):
541 541 x, y, z = self.fill_gaps(*self.decimate())
542 542 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
543 543 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
544 544 if ax.firsttime:
545 545 ax.plt = ax.pcolormesh(x, y, z[n].T,
546 546 vmin=self.zmin,
547 547 vmax=self.zmax,
548 548 cmap=plt.get_cmap(self.colormap)
549 549 )
550 550 if self.showprofile:
551 551 ax.plot_profile= self.pf_axes[n].plot(self.data['rti'][n][-1], self.y)[0]
552 552 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y,
553 553 color="k", linestyle="dashed", lw=1)[0]
554 554 else:
555 555 ax.collections.remove(ax.collections[0])
556 556 ax.plt = ax.pcolormesh(x, y, z[n].T,
557 557 vmin=self.zmin,
558 558 vmax=self.zmax,
559 559 cmap=plt.get_cmap(self.colormap)
560 560 )
561 561 if self.showprofile:
562 562 ax.plot_profile.set_data(self.data['rti'][n][-1], self.y)
563 563 ax.plot_noise.set_data(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y)
564 564
565 565 self.saveTime = self.min_time
566 566
567 567
568 568 class PlotCOHData(PlotRTIData):
569 569 '''
570 570 Plot for Coherence data
571 571 '''
572 572
573 573 CODE = 'coh'
574 574
575 575 def setup(self):
576 576 self.xaxis = 'time'
577 577 self.ncols = 1
578 578 self.nrows = len(self.data.pairs)
579 579 self.nplots = len(self.data.pairs)
580 580 self.ylabel = 'Range [Km]'
581 581 if self.CODE == 'coh':
582 582 self.cb_label = ''
583 583 self.titles = ['Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
584 584 else:
585 585 self.cb_label = 'Degrees'
586 586 self.titles = ['Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
587 587
588 588
589 589 class PlotPHASEData(PlotCOHData):
590 590 '''
591 591 Plot for Phase map data
592 592 '''
593 593
594 594 CODE = 'phase'
595 595 colormap = 'seismic'
596 596
597 597
598 598 class PlotNoiseData(PlotData):
599 599 '''
600 600 Plot for noise
601 601 '''
602 602
603 603 CODE = 'noise'
604 604
605 605 def setup(self):
606 606 self.xaxis = 'time'
607 607 self.ncols = 1
608 608 self.nrows = 1
609 609 self.nplots = 1
610 610 self.ylabel = 'Intensity [dB]'
611 611 self.titles = ['Noise']
612 612 self.colorbar = False
613 613
614 614 def plot(self):
615 615
616 616 x = self.data.times
617 617 xmin = self.min_time
618 618 xmax = xmin+self.xrange*60*60
619 619 Y = self.data[self.CODE]
620 620
621 621 if self.axes[0].firsttime:
622 622 for ch in self.data.channels:
623 623 y = Y[ch]
624 624 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
625 625 plt.legend()
626 626 else:
627 627 for ch in self.data.channels:
628 628 y = Y[ch]
629 629 self.axes[0].lines[ch].set_data(x, y)
630 630
631 631 self.ymin = numpy.nanmin(Y) - 5
632 632 self.ymax = numpy.nanmax(Y) + 5
633 633 self.saveTime = self.min_time
634 634
635 635
636 636 class PlotSNRData(PlotRTIData):
637 637 '''
638 638 Plot for SNR Data
639 639 '''
640 640
641 641 CODE = 'snr'
642 642 colormap = 'jet'
643 643
644 644
645 645 class PlotDOPData(PlotRTIData):
646 646 '''
647 647 Plot for DOPPLER Data
648 648 '''
649 649
650 650 CODE = 'dop'
651 651 colormap = 'jet'
652 652
653 653
654 654 class PlotSkyMapData(PlotData):
655 655 '''
656 656 Plot for meteors detection data
657 657 '''
658 658
659 659 CODE = 'met'
660 660
661 661 def setup(self):
662 662
663 663 self.ncols = 1
664 664 self.nrows = 1
665 665 self.width = 7.2
666 666 self.height = 7.2
667 667
668 668 self.xlabel = 'Zonal Zenith Angle (deg)'
669 669 self.ylabel = 'Meridional Zenith Angle (deg)'
670 670
671 671 if self.figure is None:
672 672 self.figure = plt.figure(figsize=(self.width, self.height),
673 673 edgecolor='k',
674 674 facecolor='w')
675 675 else:
676 676 self.figure.clf()
677 677
678 678 self.ax = plt.subplot2grid((self.nrows, self.ncols), (0, 0), 1, 1, polar=True)
679 679 self.ax.firsttime = True
680 680
681 681
682 682 def plot(self):
683 683
684 684 arrayParameters = numpy.concatenate([self.data['param'][t] for t in self.data.times])
685 685 error = arrayParameters[:,-1]
686 686 indValid = numpy.where(error == 0)[0]
687 687 finalMeteor = arrayParameters[indValid,:]
688 688 finalAzimuth = finalMeteor[:,3]
689 689 finalZenith = finalMeteor[:,4]
690 690
691 691 x = finalAzimuth*numpy.pi/180
692 692 y = finalZenith
693 693
694 694 if self.ax.firsttime:
695 695 self.ax.plot = self.ax.plot(x, y, 'bo', markersize=5)[0]
696 696 self.ax.set_ylim(0,90)
697 697 self.ax.set_yticks(numpy.arange(0,90,20))
698 698 self.ax.set_xlabel(self.xlabel)
699 699 self.ax.set_ylabel(self.ylabel)
700 700 self.ax.yaxis.labelpad = 40
701 701 self.ax.firsttime = False
702 702 else:
703 703 self.ax.plot.set_data(x, y)
704 704
705 705
706 706 dt1 = datetime.datetime.fromtimestamp(self.min_time).strftime('%y/%m/%d %H:%M:%S')
707 707 dt2 = datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')
708 708 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
709 709 dt2,
710 710 len(x))
711 711 self.ax.set_title(title, size=8)
712 712
713 713 self.saveTime = self.max_time
714 714
715 715 class PlotParamData(PlotRTIData):
716 716 '''
717 717 Plot for data_param object
718 718 '''
719 719
720 720 CODE = 'param'
721 721 colormap = 'seismic'
722 722
723 723 def setup(self):
724 724 self.xaxis = 'time'
725 725 self.ncols = 1
726 726 self.nrows = self.data.shape(self.CODE)[0]
727 727 self.nplots = self.nrows
728 728 if self.showSNR:
729 729 self.nrows += 1
730 self.nplots += 1
730 731
731 732 self.ylabel = 'Height [Km]'
732 733 self.titles = self.data.parameters \
733 734 if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
734 735 if self.showSNR:
735 736 self.titles.append('SNR')
736 737
737 738 def plot(self):
738 739 self.data.normalize_heights()
739 740 self.x = self.data.times
740 741 self.y = self.data.heights
741 742 if self.showSNR:
742 743 self.z = numpy.concatenate(
743 744 (self.data[self.CODE], self.data['snr'])
744 745 )
745 746 else:
746 747 self.z = self.data[self.CODE]
747 748
748 749 self.z = numpy.ma.masked_invalid(self.z)
749 750
750 751 for n, ax in enumerate(self.axes):
751 752
752 753 x, y, z = self.fill_gaps(*self.decimate())
753 754
754 755 if ax.firsttime:
755 756 if self.zlimits is not None:
756 757 self.zmin, self.zmax = self.zlimits[n]
757 758 self.zmax = self.zmax if self.zmax is not None else numpy.nanmax(abs(self.z[:-1, :]))
758 759 self.zmin = self.zmin if self.zmin is not None else -self.zmax
759 760 ax.plt = ax.pcolormesh(x, y, z[n, :, :].T*self.factors[n],
760 761 vmin=self.zmin,
761 762 vmax=self.zmax,
762 763 cmap=self.cmaps[n]
763 764 )
764 765 else:
765 766 if self.zlimits is not None:
766 767 self.zmin, self.zmax = self.zlimits[n]
767 768 ax.collections.remove(ax.collections[0])
768 769 ax.plt = ax.pcolormesh(x, y, z[n, :, :].T*self.factors[n],
769 770 vmin=self.zmin,
770 771 vmax=self.zmax,
771 772 cmap=self.cmaps[n]
772 773 )
773 774
774 775 self.saveTime = self.min_time
775 776
776 777 class PlotOuputData(PlotParamData):
777 778 '''
778 779 Plot data_output object
779 780 '''
780 781
781 782 CODE = 'output'
782 colormap = 'seismic' No newline at end of file
783 colormap = 'seismic'
Show file before | Show file after | Hide comments
@@ -1,2154 +1,2151
1 1 import os
2 2 import datetime
3 3 import numpy
4 4 import inspect
5 5 from figure import Figure, isRealtime, isTimeInHourRange
6 6 from plotting_codes import *
7 7
8 8
9 9 class FitGauPlot(Figure):
10 10
11 11 isConfig = None
12 12 __nsubplots = None
13 13
14 14 WIDTHPROF = None
15 15 HEIGHTPROF = None
16 16 PREFIX = 'fitgau'
17 17
18 18 def __init__(self, **kwargs):
19 19 Figure.__init__(self, **kwargs)
20 20 self.isConfig = False
21 21 self.__nsubplots = 1
22 22
23 23 self.WIDTH = 250
24 24 self.HEIGHT = 250
25 25 self.WIDTHPROF = 120
26 26 self.HEIGHTPROF = 0
27 27 self.counter_imagwr = 0
28 28
29 29 self.PLOT_CODE = SPEC_CODE
30 30
31 31 self.FTP_WEI = None
32 32 self.EXP_CODE = None
33 33 self.SUB_EXP_CODE = None
34 34 self.PLOT_POS = None
35 35
36 36 self.__xfilter_ena = False
37 37 self.__yfilter_ena = False
38 38
39 39 def getSubplots(self):
40 40
41 41 ncol = int(numpy.sqrt(self.nplots)+0.9)
42 42 nrow = int(self.nplots*1./ncol + 0.9)
43 43
44 44 return nrow, ncol
45 45
46 46 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
47 47
48 48 self.__showprofile = showprofile
49 49 self.nplots = nplots
50 50
51 51 ncolspan = 1
52 52 colspan = 1
53 53 if showprofile:
54 54 ncolspan = 3
55 55 colspan = 2
56 56 self.__nsubplots = 2
57 57
58 58 self.createFigure(id = id,
59 59 wintitle = wintitle,
60 60 widthplot = self.WIDTH + self.WIDTHPROF,
61 61 heightplot = self.HEIGHT + self.HEIGHTPROF,
62 62 show=show)
63 63
64 64 nrow, ncol = self.getSubplots()
65 65
66 66 counter = 0
67 67 for y in range(nrow):
68 68 for x in range(ncol):
69 69
70 70 if counter >= self.nplots:
71 71 break
72 72
73 73 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
74 74
75 75 if showprofile:
76 76 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
77 77
78 78 counter += 1
79 79
80 80 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
81 81 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
82 82 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
83 83 server=None, folder=None, username=None, password=None,
84 84 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
85 85 xaxis="frequency", colormap='jet', normFactor=None , GauSelector = 1):
86 86
87 87 """
88 88
89 89 Input:
90 90 dataOut :
91 91 id :
92 92 wintitle :
93 93 channelList :
94 94 showProfile :
95 95 xmin : None,
96 96 xmax : None,
97 97 ymin : None,
98 98 ymax : None,
99 99 zmin : None,
100 100 zmax : None
101 101 """
102 102 if realtime:
103 103 if not(isRealtime(utcdatatime = dataOut.utctime)):
104 104 print 'Skipping this plot function'
105 105 return
106 106
107 107 if channelList == None:
108 108 channelIndexList = dataOut.channelIndexList
109 109 else:
110 110 channelIndexList = []
111 111 for channel in channelList:
112 112 if channel not in dataOut.channelList:
113 113 raise ValueError, "Channel %d is not in dataOut.channelList" %channel
114 114 channelIndexList.append(dataOut.channelList.index(channel))
115 115
116 116 # if normFactor is None:
117 117 # factor = dataOut.normFactor
118 118 # else:
119 119 # factor = normFactor
120 120 if xaxis == "frequency":
121 121 x = dataOut.spc_range[0]
122 122 xlabel = "Frequency (kHz)"
123 123
124 124 elif xaxis == "time":
125 125 x = dataOut.spc_range[1]
126 126 xlabel = "Time (ms)"
127 127
128 128 else:
129 129 x = dataOut.spc_range[2]
130 130 xlabel = "Velocity (m/s)"
131 131
132 132 ylabel = "Range (Km)"
133 133
134 134 y = dataOut.getHeiRange()
135 135
136 136 z = dataOut.GauSPC[:,GauSelector,:,:] #GauSelector] #dataOut.data_spc/factor
137 137 print 'GausSPC', z[0,32,10:40]
138 138 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
139 139 zdB = 10*numpy.log10(z)
140 140
141 141 avg = numpy.average(z, axis=1)
142 142 avgdB = 10*numpy.log10(avg)
143 143
144 144 noise = dataOut.spc_noise
145 145 noisedB = 10*numpy.log10(noise)
146 146
147 147 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
148 148 title = wintitle + " Spectra"
149 149 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
150 150 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
151 151
152 152 if not self.isConfig:
153 153
154 154 nplots = len(channelIndexList)
155 155
156 156 self.setup(id=id,
157 157 nplots=nplots,
158 158 wintitle=wintitle,
159 159 showprofile=showprofile,
160 160 show=show)
161 161
162 162 if xmin == None: xmin = numpy.nanmin(x)
163 163 if xmax == None: xmax = numpy.nanmax(x)
164 164 if ymin == None: ymin = numpy.nanmin(y)
165 165 if ymax == None: ymax = numpy.nanmax(y)
166 166 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
167 167 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
168 168
169 169 self.FTP_WEI = ftp_wei
170 170 self.EXP_CODE = exp_code
171 171 self.SUB_EXP_CODE = sub_exp_code
172 172 self.PLOT_POS = plot_pos
173 173
174 174 self.isConfig = True
175 175
176 176 self.setWinTitle(title)
177 177
178 178 for i in range(self.nplots):
179 179 index = channelIndexList[i]
180 180 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
181 181 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[index], noisedB[index], str_datetime)
182 182 if len(dataOut.beam.codeList) != 0:
183 183 title = "Ch%d:%4.2fdB,%2.2f,%2.2f:%s" %(dataOut.channelList[index], noisedB[index], dataOut.beam.azimuthList[index], dataOut.beam.zenithList[index], str_datetime)
184 184
185 185 axes = self.axesList[i*self.__nsubplots]
186 186 axes.pcolor(x, y, zdB[index,:,:],
187 187 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
188 188 xlabel=xlabel, ylabel=ylabel, title=title, colormap=colormap,
189 189 ticksize=9, cblabel='')
190 190
191 191 if self.__showprofile:
192 192 axes = self.axesList[i*self.__nsubplots +1]
193 193 axes.pline(avgdB[index,:], y,
194 194 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
195 195 xlabel='dB', ylabel='', title='',
196 196 ytick_visible=False,
197 197 grid='x')
198 198
199 199 noiseline = numpy.repeat(noisedB[index], len(y))
200 200 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
201 201
202 202 self.draw()
203 203
204 204 if figfile == None:
205 205 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
206 206 name = str_datetime
207 207 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
208 208 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
209 209 figfile = self.getFilename(name)
210 210
211 211 self.save(figpath=figpath,
212 212 figfile=figfile,
213 213 save=save,
214 214 ftp=ftp,
215 215 wr_period=wr_period,
216 216 thisDatetime=thisDatetime)
217 217
218 218
219 219
220 220 class MomentsPlot(Figure):
221 221
222 222 isConfig = None
223 223 __nsubplots = None
224 224
225 225 WIDTHPROF = None
226 226 HEIGHTPROF = None
227 227 PREFIX = 'prm'
228 228 def __init__(self, **kwargs):
229 229 Figure.__init__(self, **kwargs)
230 230 self.isConfig = False
231 231 self.__nsubplots = 1
232 232
233 233 self.WIDTH = 280
234 234 self.HEIGHT = 250
235 235 self.WIDTHPROF = 120
236 236 self.HEIGHTPROF = 0
237 237 self.counter_imagwr = 0
238 238
239 239 self.PLOT_CODE = MOMENTS_CODE
240 240
241 241 self.FTP_WEI = None
242 242 self.EXP_CODE = None
243 243 self.SUB_EXP_CODE = None
244 244 self.PLOT_POS = None
245 245
246 246 def getSubplots(self):
247 247
248 248 ncol = int(numpy.sqrt(self.nplots)+0.9)
249 249 nrow = int(self.nplots*1./ncol + 0.9)
250 250
251 251 return nrow, ncol
252 252
253 253 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
254 254
255 255 self.__showprofile = showprofile
256 256 self.nplots = nplots
257 257
258 258 ncolspan = 1
259 259 colspan = 1
260 260 if showprofile:
261 261 ncolspan = 3
262 262 colspan = 2
263 263 self.__nsubplots = 2
264 264
265 265 self.createFigure(id = id,
266 266 wintitle = wintitle,
267 267 widthplot = self.WIDTH + self.WIDTHPROF,
268 268 heightplot = self.HEIGHT + self.HEIGHTPROF,
269 269 show=show)
270 270
271 271 nrow, ncol = self.getSubplots()
272 272
273 273 counter = 0
274 274 for y in range(nrow):
275 275 for x in range(ncol):
276 276
277 277 if counter >= self.nplots:
278 278 break
279 279
280 280 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
281 281
282 282 if showprofile:
283 283 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
284 284
285 285 counter += 1
286 286
287 287 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
288 288 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
289 289 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
290 290 server=None, folder=None, username=None, password=None,
291 291 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
292 292
293 293 """
294 294
295 295 Input:
296 296 dataOut :
297 297 id :
298 298 wintitle :
299 299 channelList :
300 300 showProfile :
301 301 xmin : None,
302 302 xmax : None,
303 303 ymin : None,
304 304 ymax : None,
305 305 zmin : None,
306 306 zmax : None
307 307 """
308 308
309 309 if dataOut.flagNoData:
310 310 return None
311 311
312 312 if realtime:
313 313 if not(isRealtime(utcdatatime = dataOut.utctime)):
314 314 print 'Skipping this plot function'
315 315 return
316 316
317 317 if channelList == None:
318 318 channelIndexList = dataOut.channelIndexList
319 319 else:
320 320 channelIndexList = []
321 321 for channel in channelList:
322 322 if channel not in dataOut.channelList:
323 323 raise ValueError, "Channel %d is not in dataOut.channelList"
324 324 channelIndexList.append(dataOut.channelList.index(channel))
325 325
326 326 factor = dataOut.normFactor
327 327 x = dataOut.abscissaList
328 328 y = dataOut.heightList
329 329
330 330 z = dataOut.data_pre[channelIndexList,:,:]/factor
331 331 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
332 332 avg = numpy.average(z, axis=1)
333 333 noise = dataOut.noise/factor
334 334
335 335 zdB = 10*numpy.log10(z)
336 336 avgdB = 10*numpy.log10(avg)
337 337 noisedB = 10*numpy.log10(noise)
338 338
339 339 #thisDatetime = dataOut.datatime
340 340 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
341 341 title = wintitle + " Parameters"
342 342 xlabel = "Velocity (m/s)"
343 343 ylabel = "Range (Km)"
344 344
345 345 update_figfile = False
346 346
347 347 if not self.isConfig:
348 348
349 349 nplots = len(channelIndexList)
350 350
351 351 self.setup(id=id,
352 352 nplots=nplots,
353 353 wintitle=wintitle,
354 354 showprofile=showprofile,
355 355 show=show)
356 356
357 357 if xmin == None: xmin = numpy.nanmin(x)
358 358 if xmax == None: xmax = numpy.nanmax(x)
359 359 if ymin == None: ymin = numpy.nanmin(y)
360 360 if ymax == None: ymax = numpy.nanmax(y)
361 361 if zmin == None: zmin = numpy.nanmin(avgdB)*0.9
362 362 if zmax == None: zmax = numpy.nanmax(avgdB)*0.9
363 363
364 364 self.FTP_WEI = ftp_wei
365 365 self.EXP_CODE = exp_code
366 366 self.SUB_EXP_CODE = sub_exp_code
367 367 self.PLOT_POS = plot_pos
368 368
369 369 self.isConfig = True
370 370 update_figfile = True
371 371
372 372 self.setWinTitle(title)
373 373
374 374 for i in range(self.nplots):
375 375 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
376 376 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[i], noisedB[i], str_datetime)
377 377 axes = self.axesList[i*self.__nsubplots]
378 378 axes.pcolor(x, y, zdB[i,:,:],
379 379 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
380 380 xlabel=xlabel, ylabel=ylabel, title=title,
381 381 ticksize=9, cblabel='')
382 382 #Mean Line
383 383 mean = dataOut.data_param[i, 1, :]
384 384 axes.addpline(mean, y, idline=0, color="black", linestyle="solid", lw=1)
385 385
386 386 if self.__showprofile:
387 387 axes = self.axesList[i*self.__nsubplots +1]
388 388 axes.pline(avgdB[i], y,
389 389 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
390 390 xlabel='dB', ylabel='', title='',
391 391 ytick_visible=False,
392 392 grid='x')
393 393
394 394 noiseline = numpy.repeat(noisedB[i], len(y))
395 395 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
396 396
397 397 self.draw()
398 398
399 399 self.save(figpath=figpath,
400 400 figfile=figfile,
401 401 save=save,
402 402 ftp=ftp,
403 403 wr_period=wr_period,
404 404 thisDatetime=thisDatetime)
405 405
406 406
407 407
408 408 class SkyMapPlot(Figure):
409 409
410 410 __isConfig = None
411 411 __nsubplots = None
412 412
413 413 WIDTHPROF = None
414 414 HEIGHTPROF = None
415 415 PREFIX = 'mmap'
416 416
417 417 def __init__(self, **kwargs):
418 418 Figure.__init__(self, **kwargs)
419 419 self.isConfig = False
420 420 self.__nsubplots = 1
421 421
422 422 # self.WIDTH = 280
423 423 # self.HEIGHT = 250
424 424 self.WIDTH = 600
425 425 self.HEIGHT = 600
426 426 self.WIDTHPROF = 120
427 427 self.HEIGHTPROF = 0
428 428 self.counter_imagwr = 0
429 429
430 430 self.PLOT_CODE = MSKYMAP_CODE
431 431
432 432 self.FTP_WEI = None
433 433 self.EXP_CODE = None
434 434 self.SUB_EXP_CODE = None
435 435 self.PLOT_POS = None
436 436
437 437 def getSubplots(self):
438 438
439 439 ncol = int(numpy.sqrt(self.nplots)+0.9)
440 440 nrow = int(self.nplots*1./ncol + 0.9)
441 441
442 442 return nrow, ncol
443 443
444 444 def setup(self, id, nplots, wintitle, showprofile=False, show=True):
445 445
446 446 self.__showprofile = showprofile
447 447 self.nplots = nplots
448 448
449 449 ncolspan = 1
450 450 colspan = 1
451 451
452 452 self.createFigure(id = id,
453 453 wintitle = wintitle,
454 454 widthplot = self.WIDTH, #+ self.WIDTHPROF,
455 455 heightplot = self.HEIGHT,# + self.HEIGHTPROF,
456 456 show=show)
457 457
458 458 nrow, ncol = 1,1
459 459 counter = 0
460 460 x = 0
461 461 y = 0
462 462 self.addAxes(1, 1, 0, 0, 1, 1, True)
463 463
464 464 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
465 465 tmin=0, tmax=24, timerange=None,
466 466 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
467 467 server=None, folder=None, username=None, password=None,
468 468 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
469 469
470 470 """
471 471
472 472 Input:
473 473 dataOut :
474 474 id :
475 475 wintitle :
476 476 channelList :
477 477 showProfile :
478 478 xmin : None,
479 479 xmax : None,
480 480 ymin : None,
481 481 ymax : None,
482 482 zmin : None,
483 483 zmax : None
484 484 """
485 485
486 486 arrayParameters = dataOut.data_param
487 487 error = arrayParameters[:,-1]
488 488 indValid = numpy.where(error == 0)[0]
489 489 finalMeteor = arrayParameters[indValid,:]
490 490 finalAzimuth = finalMeteor[:,3]
491 491 finalZenith = finalMeteor[:,4]
492 492
493 493 x = finalAzimuth*numpy.pi/180
494 494 y = finalZenith
495 495 x1 = [dataOut.ltctime, dataOut.ltctime]
496 496
497 497 #thisDatetime = dataOut.datatime
498 498 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
499 499 title = wintitle + " Parameters"
500 500 xlabel = "Zonal Zenith Angle (deg) "
501 501 ylabel = "Meridional Zenith Angle (deg)"
502 502 update_figfile = False
503 503
504 504 if not self.isConfig:
505 505
506 506 nplots = 1
507 507
508 508 self.setup(id=id,
509 509 nplots=nplots,
510 510 wintitle=wintitle,
511 511 showprofile=showprofile,
512 512 show=show)
513 513
514 514 if self.xmin is None and self.xmax is None:
515 515 self.xmin, self.xmax = self.getTimeLim(x1, tmin, tmax, timerange)
516 516
517 517 if timerange != None:
518 518 self.timerange = timerange
519 519 else:
520 520 self.timerange = self.xmax - self.xmin
521 521
522 522 self.FTP_WEI = ftp_wei
523 523 self.EXP_CODE = exp_code
524 524 self.SUB_EXP_CODE = sub_exp_code
525 525 self.PLOT_POS = plot_pos
526 526 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
527 527 self.firstdate = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
528 528 self.isConfig = True
529 529 update_figfile = True
530 530
531 531 self.setWinTitle(title)
532 532
533 533 i = 0
534 534 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
535 535
536 536 axes = self.axesList[i*self.__nsubplots]
537 537 nevents = axes.x_buffer.shape[0] + x.shape[0]
538 538 title = "Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n" %(self.firstdate,str_datetime,nevents)
539 539 axes.polar(x, y,
540 540 title=title, xlabel=xlabel, ylabel=ylabel,
541 541 ticksize=9, cblabel='')
542 542
543 543 self.draw()
544 544
545 545 self.save(figpath=figpath,
546 546 figfile=figfile,
547 547 save=save,
548 548 ftp=ftp,
549 549 wr_period=wr_period,
550 550 thisDatetime=thisDatetime,
551 551 update_figfile=update_figfile)
552 552
553 553 if dataOut.ltctime >= self.xmax:
554 554 self.isConfigmagwr = wr_period
555 555 self.isConfig = False
556 556 update_figfile = True
557 557 axes.__firsttime = True
558 558 self.xmin += self.timerange
559 559 self.xmax += self.timerange
560 560
561 561
562 562
563 563
564 564 class WindProfilerPlot(Figure):
565 565
566 566 __isConfig = None
567 567 __nsubplots = None
568 568
569 569 WIDTHPROF = None
570 570 HEIGHTPROF = None
571 571 PREFIX = 'wind'
572 572
573 573 def __init__(self, **kwargs):
574 574 Figure.__init__(self, **kwargs)
575 575 self.timerange = None
576 576 self.isConfig = False
577 577 self.__nsubplots = 1
578 578
579 579 self.WIDTH = 800
580 580 self.HEIGHT = 300
581 581 self.WIDTHPROF = 120
582 582 self.HEIGHTPROF = 0
583 583 self.counter_imagwr = 0
584 584
585 585 self.PLOT_CODE = WIND_CODE
586 586
587 587 self.FTP_WEI = None
588 588 self.EXP_CODE = None
589 589 self.SUB_EXP_CODE = None
590 590 self.PLOT_POS = None
591 591 self.tmin = None
592 592 self.tmax = None
593 593
594 594 self.xmin = None
595 595 self.xmax = None
596 596
597 597 self.figfile = None
598 598
599 599 def getSubplots(self):
600 600
601 601 ncol = 1
602 602 nrow = self.nplots
603 603
604 604 return nrow, ncol
605 605
606 606 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
607 607
608 608 self.__showprofile = showprofile
609 609 self.nplots = nplots
610 610
611 611 ncolspan = 1
612 612 colspan = 1
613 613
614 614 self.createFigure(id = id,
615 615 wintitle = wintitle,
616 616 widthplot = self.WIDTH + self.WIDTHPROF,
617 617 heightplot = self.HEIGHT + self.HEIGHTPROF,
618 618 show=show)
619 619
620 620 nrow, ncol = self.getSubplots()
621 621
622 622 counter = 0
623 623 for y in range(nrow):
624 624 if counter >= self.nplots:
625 625 break
626 626
627 627 self.addAxes(nrow, ncol*ncolspan, y, 0, colspan, 1)
628 628 counter += 1
629 629
630 630 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='False',
631 631 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
632 632 zmax_ver = None, zmin_ver = None, SNRmin = None, SNRmax = None,
633 633 timerange=None, SNRthresh = None,
634 634 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
635 635 server=None, folder=None, username=None, password=None,
636 636 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
637 637 """
638 638
639 639 Input:
640 640 dataOut :
641 641 id :
642 642 wintitle :
643 643 channelList :
644 644 showProfile :
645 645 xmin : None,
646 646 xmax : None,
647 647 ymin : None,
648 648 ymax : None,
649 649 zmin : None,
650 650 zmax : None
651 651 """
652 652
653 653 # if timerange is not None:
654 654 # self.timerange = timerange
655 655 #
656 656 # tmin = None
657 657 # tmax = None
658 658
659 659 x = dataOut.getTimeRange1(dataOut.paramInterval)
660 660 y = dataOut.heightList
661 661 z = dataOut.data_output.copy()
662 662 nplots = z.shape[0] #Number of wind dimensions estimated
663 663 nplotsw = nplots
664 664
665 665
666 666 #If there is a SNR function defined
667 667 if dataOut.data_SNR is not None:
668 668 nplots += 1
669 669 SNR = dataOut.data_SNR
670 670 SNRavg = numpy.average(SNR, axis=0)
671 671
672 672 SNRdB = 10*numpy.log10(SNR)
673 673 SNRavgdB = 10*numpy.log10(SNRavg)
674 674
675 675 if SNRthresh == None: SNRthresh = -5.0
676 676 ind = numpy.where(SNRavg < 10**(SNRthresh/10))[0]
677 677
678 678 for i in range(nplotsw):
679 679 z[i,ind] = numpy.nan
680 680
681 681 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
682 682 #thisDatetime = datetime.datetime.now()
683 683 title = wintitle + "Wind"
684 684 xlabel = ""
685 685 ylabel = "Height (km)"
686 686 update_figfile = False
687 687
688 688 if not self.isConfig:
689 689
690 690 self.setup(id=id,
691 691 nplots=nplots,
692 692 wintitle=wintitle,
693 693 showprofile=showprofile,
694 694 show=show)
695 695
696 696 if timerange is not None:
697 697 self.timerange = timerange
698 698
699 699 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
700 700
701 701 if ymin == None: ymin = numpy.nanmin(y)
702 702 if ymax == None: ymax = numpy.nanmax(y)
703 703
704 704 if zmax == None: zmax = numpy.nanmax(abs(z[range(2),:]))
705 705 #if numpy.isnan(zmax): zmax = 50
706 706 if zmin == None: zmin = -zmax
707 707
708 708 if nplotsw == 3:
709 709 if zmax_ver == None: zmax_ver = numpy.nanmax(abs(z[2,:]))
710 710 if zmin_ver == None: zmin_ver = -zmax_ver
711 711
712 712 if dataOut.data_SNR is not None:
713 713 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
714 714 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
715 715
716 716
717 717 self.FTP_WEI = ftp_wei
718 718 self.EXP_CODE = exp_code
719 719 self.SUB_EXP_CODE = sub_exp_code
720 720 self.PLOT_POS = plot_pos
721 721
722 722 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
723 723 self.isConfig = True
724 724 self.figfile = figfile
725 725 update_figfile = True
726 726
727 727 self.setWinTitle(title)
728 728
729 729 if ((self.xmax - x[1]) < (x[1]-x[0])):
730 730 x[1] = self.xmax
731 731
732 732 strWind = ['Zonal', 'Meridional', 'Vertical']
733 733 strCb = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)']
734 734 zmaxVector = [zmax, zmax, zmax_ver]
735 735 zminVector = [zmin, zmin, zmin_ver]
736 736 windFactor = [1,1,100]
737 737
738 738 for i in range(nplotsw):
739 739
740 740 title = "%s Wind: %s" %(strWind[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
741 741 axes = self.axesList[i*self.__nsubplots]
742 742
743 743 z1 = z[i,:].reshape((1,-1))*windFactor[i]
744 744 #z1=numpy.ma.masked_where(z1==0.,z1)
745 745
746 746 axes.pcolorbuffer(x, y, z1,
747 747 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zminVector[i], zmax=zmaxVector[i],
748 748 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
749 749 ticksize=9, cblabel=strCb[i], cbsize="1%", colormap="seismic" )
750 750
751 751 if dataOut.data_SNR is not None:
752 752 i += 1
753 753 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
754 754 axes = self.axesList[i*self.__nsubplots]
755 755 SNRavgdB = SNRavgdB.reshape((1,-1))
756 756 axes.pcolorbuffer(x, y, SNRavgdB,
757 757 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
758 758 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
759 759 ticksize=9, cblabel='', cbsize="1%", colormap="jet")
760 760
761 761 self.draw()
762 762
763 763 self.save(figpath=figpath,
764 764 figfile=figfile,
765 765 save=save,
766 766 ftp=ftp,
767 767 wr_period=wr_period,
768 768 thisDatetime=thisDatetime,
769 769 update_figfile=update_figfile)
770 770
771 771 if dataOut.ltctime + dataOut.paramInterval >= self.xmax:
772 772 self.counter_imagwr = wr_period
773 773 self.isConfig = False
774 774 update_figfile = True
775 775
776 776
777 777 class ParametersPlot(Figure):
778 778
779 779 __isConfig = None
780 780 __nsubplots = None
781 781
782 782 WIDTHPROF = None
783 783 HEIGHTPROF = None
784 784 PREFIX = 'param'
785 785
786 786 nplots = None
787 787 nchan = None
788 788
789 789 def __init__(self, **kwargs):
790 790 Figure.__init__(self, **kwargs)
791 791 self.timerange = None
792 792 self.isConfig = False
793 793 self.__nsubplots = 1
794 794
795 795 self.WIDTH = 800
796 796 self.HEIGHT = 180
797 797 self.WIDTHPROF = 120
798 798 self.HEIGHTPROF = 0
799 799 self.counter_imagwr = 0
800 800
801 801 self.PLOT_CODE = RTI_CODE
802 802
803 803 self.FTP_WEI = None
804 804 self.EXP_CODE = None
805 805 self.SUB_EXP_CODE = None
806 806 self.PLOT_POS = None
807 807 self.tmin = None
808 808 self.tmax = None
809 809
810 810 self.xmin = None
811 811 self.xmax = None
812 812
813 813 self.figfile = None
814 814
815 815 def getSubplots(self):
816 816
817 817 ncol = 1
818 818 nrow = self.nplots
819 819
820 820 return nrow, ncol
821 821
822 822 def setup(self, id, nplots, wintitle, show=True):
823 823
824 824 self.nplots = nplots
825 825
826 826 ncolspan = 1
827 827 colspan = 1
828 828
829 829 self.createFigure(id = id,
830 830 wintitle = wintitle,
831 831 widthplot = self.WIDTH + self.WIDTHPROF,
832 832 heightplot = self.HEIGHT + self.HEIGHTPROF,
833 833 show=show)
834 834
835 835 nrow, ncol = self.getSubplots()
836 836
837 837 counter = 0
838 838 for y in range(nrow):
839 839 for x in range(ncol):
840 840
841 841 if counter >= self.nplots:
842 842 break
843 843
844 844 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
845 845
846 846 counter += 1
847 847
848 848 def run(self, dataOut, id, wintitle="", channelList=None, paramIndex = 0, colormap="jet",
849 849 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None, timerange=None,
850 850 showSNR=False, SNRthresh = -numpy.inf, SNRmin=None, SNRmax=None,
851 851 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
852 852 server=None, folder=None, username=None, password=None,
853 853 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, HEIGHT=None):
854 854 """
855 855
856 856 Input:
857 857 dataOut :
858 858 id :
859 859 wintitle :
860 860 channelList :
861 861 showProfile :
862 862 xmin : None,
863 863 xmax : None,
864 864 ymin : None,
865 865 ymax : None,
866 866 zmin : None,
867 867 zmax : None
868 868 """
869 869
870 870 if HEIGHT is not None:
871 871 self.HEIGHT = HEIGHT
872 872
873 873
874 874 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
875 875 return
876 876
877 877 if channelList == None:
878 878 channelIndexList = range(dataOut.data_param.shape[0])
879 879 else:
880 880 channelIndexList = []
881 881 for channel in channelList:
882 882 if channel not in dataOut.channelList:
883 883 raise ValueError, "Channel %d is not in dataOut.channelList"
884 884 channelIndexList.append(dataOut.channelList.index(channel))
885 885
886 886 x = dataOut.getTimeRange1(dataOut.paramInterval)
887 887 y = dataOut.getHeiRange()
888 888
889 889 if dataOut.data_param.ndim == 3:
890 890 z = dataOut.data_param[channelIndexList,paramIndex,:]
891 891 else:
892 892 z = dataOut.data_param[channelIndexList,:]
893 893
894 894 if showSNR:
895 895 #SNR data
896 896 SNRarray = dataOut.data_SNR[channelIndexList,:]
897 897 SNRdB = 10*numpy.log10(SNRarray)
898 898 ind = numpy.where(SNRdB < SNRthresh)
899 899 z[ind] = numpy.nan
900 900
901 901 thisDatetime = dataOut.datatime
902 902 # thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
903 903 title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
904 904 xlabel = ""
905 905 ylabel = "Range (Km)"
906 906
907 907 update_figfile = False
908 908
909 909 if not self.isConfig:
910 910
911 911 nchan = len(channelIndexList)
912 912 self.nchan = nchan
913 913 self.plotFact = 1
914 914 nplots = nchan
915 915
916 916 if showSNR:
917 917 nplots = nchan*2
918 918 self.plotFact = 2
919 919 if SNRmin == None: SNRmin = numpy.nanmin(SNRdB)
920 920 if SNRmax == None: SNRmax = numpy.nanmax(SNRdB)
921 921
922 922 self.setup(id=id,
923 923 nplots=nplots,
924 924 wintitle=wintitle,
925 925 show=show)
926 926
927 927 if timerange != None:
928 928 self.timerange = timerange
929 929
930 930 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
931 931
932 932 if ymin == None: ymin = numpy.nanmin(y)
933 933 if ymax == None: ymax = numpy.nanmax(y)
934 934 if zmin == None: zmin = numpy.nanmin(z)
935 935 if zmax == None: zmax = numpy.nanmax(z)
936 936
937 937 self.FTP_WEI = ftp_wei
938 938 self.EXP_CODE = exp_code
939 939 self.SUB_EXP_CODE = sub_exp_code
940 940 self.PLOT_POS = plot_pos
941 941
942 942 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
943 943 self.isConfig = True
944 944 self.figfile = figfile
945 945 update_figfile = True
946 946
947 947 self.setWinTitle(title)
948 948
949 949 for i in range(self.nchan):
950 950 index = channelIndexList[i]
951 951 title = "Channel %d: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
952 952 axes = self.axesList[i*self.plotFact]
953 953 z1 = z[i,:].reshape((1,-1))
954 954 axes.pcolorbuffer(x, y, z1,
955 955 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
956 956 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
957 957 ticksize=9, cblabel='', cbsize="1%",colormap=colormap)
958 958
959 959 if showSNR:
960 960 title = "Channel %d SNR: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
961 961 axes = self.axesList[i*self.plotFact + 1]
962 962 SNRdB1 = SNRdB[i,:].reshape((1,-1))
963 963 axes.pcolorbuffer(x, y, SNRdB1,
964 964 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
965 965 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
966 966 ticksize=9, cblabel='', cbsize="1%",colormap='jet')
967 967
968 968
969 969 self.draw()
970 970
971 971 if dataOut.ltctime >= self.xmax:
972 972 self.counter_imagwr = wr_period
973 973 self.isConfig = False
974 974 update_figfile = True
975 975
976 976 self.save(figpath=figpath,
977 977 figfile=figfile,
978 978 save=save,
979 979 ftp=ftp,
980 980 wr_period=wr_period,
981 981 thisDatetime=thisDatetime,
982 982 update_figfile=update_figfile)
983 983
984 984
985 985
986 986 class Parameters1Plot(Figure):
987 987
988 988 __isConfig = None
989 989 __nsubplots = None
990 990
991 991 WIDTHPROF = None
992 992 HEIGHTPROF = None
993 993 PREFIX = 'prm'
994 994
995 995 def __init__(self, **kwargs):
996 996 Figure.__init__(self, **kwargs)
997 997 self.timerange = 2*60*60
998 998 self.isConfig = False
999 999 self.__nsubplots = 1
1000 1000
1001 1001 self.WIDTH = 800
1002 1002 self.HEIGHT = 180
1003 1003 self.WIDTHPROF = 120
1004 1004 self.HEIGHTPROF = 0
1005 1005 self.counter_imagwr = 0
1006 1006
1007 1007 self.PLOT_CODE = PARMS_CODE
1008 1008
1009 1009 self.FTP_WEI = None
1010 1010 self.EXP_CODE = None
1011 1011 self.SUB_EXP_CODE = None
1012 1012 self.PLOT_POS = None
1013 1013 self.tmin = None
1014 1014 self.tmax = None
1015 1015
1016 1016 self.xmin = None
1017 1017 self.xmax = None
1018 1018
1019 1019 self.figfile = None
1020 1020
1021 1021 def getSubplots(self):
1022 1022
1023 1023 ncol = 1
1024 1024 nrow = self.nplots
1025 1025
1026 1026 return nrow, ncol
1027 1027
1028 1028 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1029 1029
1030 1030 self.__showprofile = showprofile
1031 1031 self.nplots = nplots
1032 1032
1033 1033 ncolspan = 1
1034 1034 colspan = 1
1035 1035
1036 1036 self.createFigure(id = id,
1037 1037 wintitle = wintitle,
1038 1038 widthplot = self.WIDTH + self.WIDTHPROF,
1039 1039 heightplot = self.HEIGHT + self.HEIGHTPROF,
1040 1040 show=show)
1041 1041
1042 1042 nrow, ncol = self.getSubplots()
1043 1043
1044 1044 counter = 0
1045 1045 for y in range(nrow):
1046 1046 for x in range(ncol):
1047 1047
1048 1048 if counter >= self.nplots:
1049 1049 break
1050 1050
1051 1051 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1052 1052
1053 1053 if showprofile:
1054 1054 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
1055 1055
1056 1056 counter += 1
1057 1057
1058 1058 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
1059 1059 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,timerange=None,
1060 1060 parameterIndex = None, onlyPositive = False,
1061 1061 SNRthresh = -numpy.inf, SNR = True, SNRmin = None, SNRmax = None, onlySNR = False,
1062 1062 DOP = True,
1063 1063 zlabel = "", parameterName = "", parameterObject = "data_param",
1064 1064 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
1065 1065 server=None, folder=None, username=None, password=None,
1066 1066 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1067 1067 #print inspect.getargspec(self.run).args
1068 1068 """
1069 1069
1070 1070 Input:
1071 1071 dataOut :
1072 1072 id :
1073 1073 wintitle :
1074 1074 channelList :
1075 1075 showProfile :
1076 1076 xmin : None,
1077 1077 xmax : None,
1078 1078 ymin : None,
1079 1079 ymax : None,
1080 1080 zmin : None,
1081 1081 zmax : None
1082 1082 """
1083 1083
1084 1084 data_param = getattr(dataOut, parameterObject)
1085 1085
1086 1086 if channelList == None:
1087 1087 channelIndexList = numpy.arange(data_param.shape[0])
1088 1088 else:
1089 1089 channelIndexList = numpy.array(channelList)
1090 1090
1091 1091 nchan = len(channelIndexList) #Number of channels being plotted
1092 1092
1093 1093 if nchan < 1:
1094 1094 return
1095 1095
1096 1096 nGraphsByChannel = 0
1097 1097
1098 1098 if SNR:
1099 1099 nGraphsByChannel += 1
1100 1100 if DOP:
1101 1101 nGraphsByChannel += 1
1102 1102
1103 1103 if nGraphsByChannel < 1:
1104 1104 return
1105 1105
1106 1106 nplots = nGraphsByChannel*nchan
1107 1107
1108 1108 if timerange is not None:
1109 1109 self.timerange = timerange
1110 1110
1111 1111 #tmin = None
1112 1112 #tmax = None
1113 1113 if parameterIndex == None:
1114 1114 parameterIndex = 1
1115 1115
1116 1116 x = dataOut.getTimeRange1(dataOut.paramInterval)
1117 1117 y = dataOut.heightList
1118 z = data_param[channelIndexList,parameterIndex,:].copy()
1119 1118
1120 zRange = dataOut.abscissaList
1121 # nChannels = z.shape[0] #Number of wind dimensions estimated
1122 # thisDatetime = dataOut.datatime
1119 if dataOut.data_param.ndim == 3:
1120 z = dataOut.data_param[channelIndexList,parameterIndex,:]
1121 else:
1122 z = dataOut.data_param[channelIndexList,:]
1123 1123
1124 1124 if dataOut.data_SNR is not None:
1125 SNRarray = dataOut.data_SNR[channelIndexList,:]
1126 SNRdB = 10*numpy.log10(SNRarray)
1127 # SNRavgdB = 10*numpy.log10(SNRavg)
1128 ind = numpy.where(SNRdB < 10**(SNRthresh/10))
1129 z[ind] = numpy.nan
1125 if dataOut.data_SNR.ndim == 2:
1126 SNRavg = numpy.average(dataOut.data_SNR, axis=0)
1127 else:
1128 SNRavg = dataOut.data_SNR
1129 SNRdB = 10*numpy.log10(SNRavg)
1130 1130
1131 1131 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1132 1132 title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
1133 1133 xlabel = ""
1134 ylabel = "Range (Km)"
1135
1136 if (SNR and not onlySNR): nplots = 2*nplots
1134 ylabel = "Range (Km)"
1137 1135
1138 1136 if onlyPositive:
1139 1137 colormap = "jet"
1140 1138 zmin = 0
1141 1139 else: colormap = "RdBu_r"
1142 1140
1143 1141 if not self.isConfig:
1144 1142
1145 1143 self.setup(id=id,
1146 1144 nplots=nplots,
1147 1145 wintitle=wintitle,
1148 1146 showprofile=showprofile,
1149 1147 show=show)
1150 1148
1151 1149 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1152 1150
1153 1151 if ymin == None: ymin = numpy.nanmin(y)
1154 1152 if ymax == None: ymax = numpy.nanmax(y)
1155 if zmin == None: zmin = numpy.nanmin(zRange)
1156 if zmax == None: zmax = numpy.nanmax(zRange)
1153 if zmin == None: zmin = numpy.nanmin(z)
1154 if zmax == None: zmax = numpy.nanmax(z)
1157 1155
1158 1156 if SNR:
1159 1157 if SNRmin == None: SNRmin = numpy.nanmin(SNRdB)
1160 1158 if SNRmax == None: SNRmax = numpy.nanmax(SNRdB)
1161 1159
1162 1160 self.FTP_WEI = ftp_wei
1163 1161 self.EXP_CODE = exp_code
1164 1162 self.SUB_EXP_CODE = sub_exp_code
1165 1163 self.PLOT_POS = plot_pos
1166 1164
1167 1165 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1168 1166 self.isConfig = True
1169 1167 self.figfile = figfile
1170 1168
1171 1169 self.setWinTitle(title)
1172 1170
1173 1171 if ((self.xmax - x[1]) < (x[1]-x[0])):
1174 1172 x[1] = self.xmax
1175 1173
1176 1174 for i in range(nchan):
1177 1175
1178 1176 if (SNR and not onlySNR): j = 2*i
1179 1177 else: j = i
1180 1178
1181 1179 j = nGraphsByChannel*i
1182 1180
1183 1181 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
1184 1182 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
1185 1183
1186 1184 if not onlySNR:
1187 1185 axes = self.axesList[j*self.__nsubplots]
1188 1186 z1 = z[i,:].reshape((1,-1))
1189 1187 axes.pcolorbuffer(x, y, z1,
1190 1188 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
1191 1189 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
1192 1190 ticksize=9, cblabel=zlabel, cbsize="1%")
1193 1191
1194 1192 if DOP:
1195 1193 title = "%s Channel %d: %s" %(parameterName, channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1196 1194
1197 1195 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
1198 1196 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
1199 1197 axes = self.axesList[j]
1200 1198 z1 = z[i,:].reshape((1,-1))
1201 1199 axes.pcolorbuffer(x, y, z1,
1202 1200 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
1203 1201 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
1204 1202 ticksize=9, cblabel=zlabel, cbsize="1%")
1205 1203
1206 if SNR:
1207 title = "Channel %d Signal Noise Ratio (SNR): %s" %(channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1208 axes = self.axesList[(j)*self.__nsubplots]
1209 if not onlySNR:
1210 axes = self.axesList[(j + 1)*self.__nsubplots]
1211
1212 axes = self.axesList[(j + nGraphsByChannel-1)]
1204 if SNR:
1205 title = "Channel %d Signal Noise Ratio (SNR): %s" %(channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1206 axes = self.axesList[(j)*self.__nsubplots]
1207 if not onlySNR:
1208 axes = self.axesList[(j + 1)*self.__nsubplots]
1213 1209
1214 z1 = SNRdB[i,:].reshape((1,-1))
1215 axes.pcolorbuffer(x, y, z1,
1216 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
1217 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap="jet",
1218 ticksize=9, cblabel=zlabel, cbsize="1%")
1210 axes = self.axesList[(j + nGraphsByChannel-1)]
1211 z1 = SNRdB.reshape((1,-1))
1212 axes.pcolorbuffer(x, y, z1,
1213 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
1214 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap="jet",
1215 ticksize=9, cblabel=zlabel, cbsize="1%")
1219 1216
1220 1217
1221 1218
1222 1219 self.draw()
1223 1220
1224 1221 if x[1] >= self.axesList[0].xmax:
1225 1222 self.counter_imagwr = wr_period
1226 1223 self.isConfig = False
1227 1224 self.figfile = None
1228 1225
1229 1226 self.save(figpath=figpath,
1230 1227 figfile=figfile,
1231 1228 save=save,
1232 1229 ftp=ftp,
1233 1230 wr_period=wr_period,
1234 1231 thisDatetime=thisDatetime,
1235 1232 update_figfile=False)
1236 1233
1237 1234 class SpectralFittingPlot(Figure):
1238 1235
1239 1236 __isConfig = None
1240 1237 __nsubplots = None
1241 1238
1242 1239 WIDTHPROF = None
1243 1240 HEIGHTPROF = None
1244 1241 PREFIX = 'prm'
1245 1242
1246 1243
1247 1244 N = None
1248 1245 ippSeconds = None
1249 1246
1250 1247 def __init__(self, **kwargs):
1251 1248 Figure.__init__(self, **kwargs)
1252 1249 self.isConfig = False
1253 1250 self.__nsubplots = 1
1254 1251
1255 1252 self.PLOT_CODE = SPECFIT_CODE
1256 1253
1257 1254 self.WIDTH = 450
1258 1255 self.HEIGHT = 250
1259 1256 self.WIDTHPROF = 0
1260 1257 self.HEIGHTPROF = 0
1261 1258
1262 1259 def getSubplots(self):
1263 1260
1264 1261 ncol = int(numpy.sqrt(self.nplots)+0.9)
1265 1262 nrow = int(self.nplots*1./ncol + 0.9)
1266 1263
1267 1264 return nrow, ncol
1268 1265
1269 1266 def setup(self, id, nplots, wintitle, showprofile=False, show=True):
1270 1267
1271 1268 showprofile = False
1272 1269 self.__showprofile = showprofile
1273 1270 self.nplots = nplots
1274 1271
1275 1272 ncolspan = 5
1276 1273 colspan = 4
1277 1274 if showprofile:
1278 1275 ncolspan = 5
1279 1276 colspan = 4
1280 1277 self.__nsubplots = 2
1281 1278
1282 1279 self.createFigure(id = id,
1283 1280 wintitle = wintitle,
1284 1281 widthplot = self.WIDTH + self.WIDTHPROF,
1285 1282 heightplot = self.HEIGHT + self.HEIGHTPROF,
1286 1283 show=show)
1287 1284
1288 1285 nrow, ncol = self.getSubplots()
1289 1286
1290 1287 counter = 0
1291 1288 for y in range(nrow):
1292 1289 for x in range(ncol):
1293 1290
1294 1291 if counter >= self.nplots:
1295 1292 break
1296 1293
1297 1294 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1298 1295
1299 1296 if showprofile:
1300 1297 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
1301 1298
1302 1299 counter += 1
1303 1300
1304 1301 def run(self, dataOut, id, cutHeight=None, fit=False, wintitle="", channelList=None, showprofile=True,
1305 1302 xmin=None, xmax=None, ymin=None, ymax=None,
1306 1303 save=False, figpath='./', figfile=None, show=True):
1307 1304
1308 1305 """
1309 1306
1310 1307 Input:
1311 1308 dataOut :
1312 1309 id :
1313 1310 wintitle :
1314 1311 channelList :
1315 1312 showProfile :
1316 1313 xmin : None,
1317 1314 xmax : None,
1318 1315 zmin : None,
1319 1316 zmax : None
1320 1317 """
1321 1318
1322 1319 if cutHeight==None:
1323 1320 h=270
1324 1321 heightindex = numpy.abs(cutHeight - dataOut.heightList).argmin()
1325 1322 cutHeight = dataOut.heightList[heightindex]
1326 1323
1327 1324 factor = dataOut.normFactor
1328 1325 x = dataOut.abscissaList[:-1]
1329 1326 #y = dataOut.getHeiRange()
1330 1327
1331 1328 z = dataOut.data_pre[:,:,heightindex]/factor
1332 1329 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
1333 1330 avg = numpy.average(z, axis=1)
1334 1331 listChannels = z.shape[0]
1335 1332
1336 1333 #Reconstruct Function
1337 1334 if fit==True:
1338 1335 groupArray = dataOut.groupList
1339 1336 listChannels = groupArray.reshape((groupArray.size))
1340 1337 listChannels.sort()
1341 1338 spcFitLine = numpy.zeros(z.shape)
1342 1339 constants = dataOut.constants
1343 1340
1344 1341 nGroups = groupArray.shape[0]
1345 1342 nChannels = groupArray.shape[1]
1346 1343 nProfiles = z.shape[1]
1347 1344
1348 1345 for f in range(nGroups):
1349 1346 groupChann = groupArray[f,:]
1350 1347 p = dataOut.data_param[f,:,heightindex]
1351 1348 # p = numpy.array([ 89.343967,0.14036615,0.17086219,18.89835291,1.58388365,1.55099167])
1352 1349 fitLineAux = dataOut.library.modelFunction(p, constants)*nProfiles
1353 1350 fitLineAux = fitLineAux.reshape((nChannels,nProfiles))
1354 1351 spcFitLine[groupChann,:] = fitLineAux
1355 1352 # spcFitLine = spcFitLine/factor
1356 1353
1357 1354 z = z[listChannels,:]
1358 1355 spcFitLine = spcFitLine[listChannels,:]
1359 1356 spcFitLinedB = 10*numpy.log10(spcFitLine)
1360 1357
1361 1358 zdB = 10*numpy.log10(z)
1362 1359 #thisDatetime = dataOut.datatime
1363 1360 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1364 1361 title = wintitle + " Doppler Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
1365 1362 xlabel = "Velocity (m/s)"
1366 1363 ylabel = "Spectrum"
1367 1364
1368 1365 if not self.isConfig:
1369 1366
1370 1367 nplots = listChannels.size
1371 1368
1372 1369 self.setup(id=id,
1373 1370 nplots=nplots,
1374 1371 wintitle=wintitle,
1375 1372 showprofile=showprofile,
1376 1373 show=show)
1377 1374
1378 1375 if xmin == None: xmin = numpy.nanmin(x)
1379 1376 if xmax == None: xmax = numpy.nanmax(x)
1380 1377 if ymin == None: ymin = numpy.nanmin(zdB)
1381 1378 if ymax == None: ymax = numpy.nanmax(zdB)+2
1382 1379
1383 1380 self.isConfig = True
1384 1381
1385 1382 self.setWinTitle(title)
1386 1383 for i in range(self.nplots):
1387 1384 # title = "Channel %d: %4.2fdB" %(dataOut.channelList[i]+1, noisedB[i])
1388 1385 title = "Height %4.1f km\nChannel %d:" %(cutHeight, listChannels[i])
1389 1386 axes = self.axesList[i*self.__nsubplots]
1390 1387 if fit == False:
1391 1388 axes.pline(x, zdB[i,:],
1392 1389 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1393 1390 xlabel=xlabel, ylabel=ylabel, title=title
1394 1391 )
1395 1392 if fit == True:
1396 1393 fitline=spcFitLinedB[i,:]
1397 1394 y=numpy.vstack([zdB[i,:],fitline] )
1398 1395 legendlabels=['Data','Fitting']
1399 1396 axes.pmultilineyaxis(x, y,
1400 1397 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1401 1398 xlabel=xlabel, ylabel=ylabel, title=title,
1402 1399 legendlabels=legendlabels, marker=None,
1403 1400 linestyle='solid', grid='both')
1404 1401
1405 1402 self.draw()
1406 1403
1407 1404 self.save(figpath=figpath,
1408 1405 figfile=figfile,
1409 1406 save=save,
1410 1407 ftp=ftp,
1411 1408 wr_period=wr_period,
1412 1409 thisDatetime=thisDatetime)
1413 1410
1414 1411
1415 1412 class EWDriftsPlot(Figure):
1416 1413
1417 1414 __isConfig = None
1418 1415 __nsubplots = None
1419 1416
1420 1417 WIDTHPROF = None
1421 1418 HEIGHTPROF = None
1422 1419 PREFIX = 'drift'
1423 1420
1424 1421 def __init__(self, **kwargs):
1425 1422 Figure.__init__(self, **kwargs)
1426 1423 self.timerange = 2*60*60
1427 1424 self.isConfig = False
1428 1425 self.__nsubplots = 1
1429 1426
1430 1427 self.WIDTH = 800
1431 1428 self.HEIGHT = 150
1432 1429 self.WIDTHPROF = 120
1433 1430 self.HEIGHTPROF = 0
1434 1431 self.counter_imagwr = 0
1435 1432
1436 1433 self.PLOT_CODE = EWDRIFT_CODE
1437 1434
1438 1435 self.FTP_WEI = None
1439 1436 self.EXP_CODE = None
1440 1437 self.SUB_EXP_CODE = None
1441 1438 self.PLOT_POS = None
1442 1439 self.tmin = None
1443 1440 self.tmax = None
1444 1441
1445 1442 self.xmin = None
1446 1443 self.xmax = None
1447 1444
1448 1445 self.figfile = None
1449 1446
1450 1447 def getSubplots(self):
1451 1448
1452 1449 ncol = 1
1453 1450 nrow = self.nplots
1454 1451
1455 1452 return nrow, ncol
1456 1453
1457 1454 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1458 1455
1459 1456 self.__showprofile = showprofile
1460 1457 self.nplots = nplots
1461 1458
1462 1459 ncolspan = 1
1463 1460 colspan = 1
1464 1461
1465 1462 self.createFigure(id = id,
1466 1463 wintitle = wintitle,
1467 1464 widthplot = self.WIDTH + self.WIDTHPROF,
1468 1465 heightplot = self.HEIGHT + self.HEIGHTPROF,
1469 1466 show=show)
1470 1467
1471 1468 nrow, ncol = self.getSubplots()
1472 1469
1473 1470 counter = 0
1474 1471 for y in range(nrow):
1475 1472 if counter >= self.nplots:
1476 1473 break
1477 1474
1478 1475 self.addAxes(nrow, ncol*ncolspan, y, 0, colspan, 1)
1479 1476 counter += 1
1480 1477
1481 1478 def run(self, dataOut, id, wintitle="", channelList=None,
1482 1479 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
1483 1480 zmaxVertical = None, zminVertical = None, zmaxZonal = None, zminZonal = None,
1484 1481 timerange=None, SNRthresh = -numpy.inf, SNRmin = None, SNRmax = None, SNR_1 = False,
1485 1482 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
1486 1483 server=None, folder=None, username=None, password=None,
1487 1484 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1488 1485 """
1489 1486
1490 1487 Input:
1491 1488 dataOut :
1492 1489 id :
1493 1490 wintitle :
1494 1491 channelList :
1495 1492 showProfile :
1496 1493 xmin : None,
1497 1494 xmax : None,
1498 1495 ymin : None,
1499 1496 ymax : None,
1500 1497 zmin : None,
1501 1498 zmax : None
1502 1499 """
1503 1500
1504 1501 if timerange is not None:
1505 1502 self.timerange = timerange
1506 1503
1507 1504 tmin = None
1508 1505 tmax = None
1509 1506
1510 1507 x = dataOut.getTimeRange1(dataOut.outputInterval)
1511 1508 # y = dataOut.heightList
1512 1509 y = dataOut.heightList
1513 1510
1514 1511 z = dataOut.data_output
1515 1512 nplots = z.shape[0] #Number of wind dimensions estimated
1516 1513 nplotsw = nplots
1517 1514
1518 1515 #If there is a SNR function defined
1519 1516 if dataOut.data_SNR is not None:
1520 1517 nplots += 1
1521 1518 SNR = dataOut.data_SNR
1522 1519
1523 1520 if SNR_1:
1524 1521 SNR += 1
1525 1522
1526 1523 SNRavg = numpy.average(SNR, axis=0)
1527 1524
1528 1525 SNRdB = 10*numpy.log10(SNR)
1529 1526 SNRavgdB = 10*numpy.log10(SNRavg)
1530 1527
1531 1528 ind = numpy.where(SNRavg < 10**(SNRthresh/10))[0]
1532 1529
1533 1530 for i in range(nplotsw):
1534 1531 z[i,ind] = numpy.nan
1535 1532
1536 1533
1537 1534 showprofile = False
1538 1535 # thisDatetime = dataOut.datatime
1539 1536 thisDatetime = datetime.datetime.utcfromtimestamp(x[1])
1540 1537 title = wintitle + " EW Drifts"
1541 1538 xlabel = ""
1542 1539 ylabel = "Height (Km)"
1543 1540
1544 1541 if not self.isConfig:
1545 1542
1546 1543 self.setup(id=id,
1547 1544 nplots=nplots,
1548 1545 wintitle=wintitle,
1549 1546 showprofile=showprofile,
1550 1547 show=show)
1551 1548
1552 1549 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1553 1550
1554 1551 if ymin == None: ymin = numpy.nanmin(y)
1555 1552 if ymax == None: ymax = numpy.nanmax(y)
1556 1553
1557 1554 if zmaxZonal == None: zmaxZonal = numpy.nanmax(abs(z[0,:]))
1558 1555 if zminZonal == None: zminZonal = -zmaxZonal
1559 1556 if zmaxVertical == None: zmaxVertical = numpy.nanmax(abs(z[1,:]))
1560 1557 if zminVertical == None: zminVertical = -zmaxVertical
1561 1558
1562 1559 if dataOut.data_SNR is not None:
1563 1560 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
1564 1561 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
1565 1562
1566 1563 self.FTP_WEI = ftp_wei
1567 1564 self.EXP_CODE = exp_code
1568 1565 self.SUB_EXP_CODE = sub_exp_code
1569 1566 self.PLOT_POS = plot_pos
1570 1567
1571 1568 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1572 1569 self.isConfig = True
1573 1570
1574 1571
1575 1572 self.setWinTitle(title)
1576 1573
1577 1574 if ((self.xmax - x[1]) < (x[1]-x[0])):
1578 1575 x[1] = self.xmax
1579 1576
1580 1577 strWind = ['Zonal','Vertical']
1581 1578 strCb = 'Velocity (m/s)'
1582 1579 zmaxVector = [zmaxZonal, zmaxVertical]
1583 1580 zminVector = [zminZonal, zminVertical]
1584 1581
1585 1582 for i in range(nplotsw):
1586 1583
1587 1584 title = "%s Drifts: %s" %(strWind[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1588 1585 axes = self.axesList[i*self.__nsubplots]
1589 1586
1590 1587 z1 = z[i,:].reshape((1,-1))
1591 1588
1592 1589 axes.pcolorbuffer(x, y, z1,
1593 1590 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zminVector[i], zmax=zmaxVector[i],
1594 1591 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
1595 1592 ticksize=9, cblabel=strCb, cbsize="1%", colormap="RdBu_r")
1596 1593
1597 1594 if dataOut.data_SNR is not None:
1598 1595 i += 1
1599 1596 if SNR_1:
1600 1597 title = "Signal Noise Ratio + 1 (SNR+1): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1601 1598 else:
1602 1599 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1603 1600 axes = self.axesList[i*self.__nsubplots]
1604 1601 SNRavgdB = SNRavgdB.reshape((1,-1))
1605 1602
1606 1603 axes.pcolorbuffer(x, y, SNRavgdB,
1607 1604 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
1608 1605 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
1609 1606 ticksize=9, cblabel='', cbsize="1%", colormap="jet")
1610 1607
1611 1608 self.draw()
1612 1609
1613 1610 if x[1] >= self.axesList[0].xmax:
1614 1611 self.counter_imagwr = wr_period
1615 1612 self.isConfig = False
1616 1613 self.figfile = None
1617 1614
1618 1615
1619 1616
1620 1617
1621 1618 class PhasePlot(Figure):
1622 1619
1623 1620 __isConfig = None
1624 1621 __nsubplots = None
1625 1622
1626 1623 PREFIX = 'mphase'
1627 1624
1628 1625
1629 1626 def __init__(self, **kwargs):
1630 1627 Figure.__init__(self, **kwargs)
1631 1628 self.timerange = 24*60*60
1632 1629 self.isConfig = False
1633 1630 self.__nsubplots = 1
1634 1631 self.counter_imagwr = 0
1635 1632 self.WIDTH = 600
1636 1633 self.HEIGHT = 300
1637 1634 self.WIDTHPROF = 120
1638 1635 self.HEIGHTPROF = 0
1639 1636 self.xdata = None
1640 1637 self.ydata = None
1641 1638
1642 1639 self.PLOT_CODE = MPHASE_CODE
1643 1640
1644 1641 self.FTP_WEI = None
1645 1642 self.EXP_CODE = None
1646 1643 self.SUB_EXP_CODE = None
1647 1644 self.PLOT_POS = None
1648 1645
1649 1646
1650 1647 self.filename_phase = None
1651 1648
1652 1649 self.figfile = None
1653 1650
1654 1651 def getSubplots(self):
1655 1652
1656 1653 ncol = 1
1657 1654 nrow = 1
1658 1655
1659 1656 return nrow, ncol
1660 1657
1661 1658 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1662 1659
1663 1660 self.__showprofile = showprofile
1664 1661 self.nplots = nplots
1665 1662
1666 1663 ncolspan = 7
1667 1664 colspan = 6
1668 1665 self.__nsubplots = 2
1669 1666
1670 1667 self.createFigure(id = id,
1671 1668 wintitle = wintitle,
1672 1669 widthplot = self.WIDTH+self.WIDTHPROF,
1673 1670 heightplot = self.HEIGHT+self.HEIGHTPROF,
1674 1671 show=show)
1675 1672
1676 1673 nrow, ncol = self.getSubplots()
1677 1674
1678 1675 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1679 1676
1680 1677
1681 1678 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
1682 1679 xmin=None, xmax=None, ymin=None, ymax=None,
1683 1680 timerange=None,
1684 1681 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1685 1682 server=None, folder=None, username=None, password=None,
1686 1683 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1687 1684
1688 1685
1689 1686 tmin = None
1690 1687 tmax = None
1691 1688 x = dataOut.getTimeRange1(dataOut.outputInterval)
1692 1689 y = dataOut.getHeiRange()
1693 1690
1694 1691
1695 1692 #thisDatetime = dataOut.datatime
1696 1693 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
1697 1694 title = wintitle + " Phase of Beacon Signal" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1698 1695 xlabel = "Local Time"
1699 1696 ylabel = "Phase"
1700 1697
1701 1698
1702 1699 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
1703 1700 phase_beacon = dataOut.data_output
1704 1701 update_figfile = False
1705 1702
1706 1703 if not self.isConfig:
1707 1704
1708 1705 self.nplots = phase_beacon.size
1709 1706
1710 1707 self.setup(id=id,
1711 1708 nplots=self.nplots,
1712 1709 wintitle=wintitle,
1713 1710 showprofile=showprofile,
1714 1711 show=show)
1715 1712
1716 1713 if timerange is not None:
1717 1714 self.timerange = timerange
1718 1715
1719 1716 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1720 1717
1721 1718 if ymin == None: ymin = numpy.nanmin(phase_beacon) - 10.0
1722 1719 if ymax == None: ymax = numpy.nanmax(phase_beacon) + 10.0
1723 1720
1724 1721 self.FTP_WEI = ftp_wei
1725 1722 self.EXP_CODE = exp_code
1726 1723 self.SUB_EXP_CODE = sub_exp_code
1727 1724 self.PLOT_POS = plot_pos
1728 1725
1729 1726 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1730 1727 self.isConfig = True
1731 1728 self.figfile = figfile
1732 1729 self.xdata = numpy.array([])
1733 1730 self.ydata = numpy.array([])
1734 1731
1735 1732 #open file beacon phase
1736 1733 path = '%s%03d' %(self.PREFIX, self.id)
1737 1734 beacon_file = os.path.join(path,'%s.txt'%self.name)
1738 1735 self.filename_phase = os.path.join(figpath,beacon_file)
1739 1736 update_figfile = True
1740 1737
1741 1738
1742 1739 #store data beacon phase
1743 1740 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
1744 1741
1745 1742 self.setWinTitle(title)
1746 1743
1747 1744
1748 1745 title = "Phase Offset %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1749 1746
1750 1747 legendlabels = ["phase %d"%(chan) for chan in numpy.arange(self.nplots)]
1751 1748
1752 1749 axes = self.axesList[0]
1753 1750
1754 1751 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1755 1752
1756 1753 if len(self.ydata)==0:
1757 1754 self.ydata = phase_beacon.reshape(-1,1)
1758 1755 else:
1759 1756 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
1760 1757
1761 1758
1762 1759 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1763 1760 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1764 1761 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1765 1762 XAxisAsTime=True, grid='both'
1766 1763 )
1767 1764
1768 1765 self.draw()
1769 1766
1770 1767 self.save(figpath=figpath,
1771 1768 figfile=figfile,
1772 1769 save=save,
1773 1770 ftp=ftp,
1774 1771 wr_period=wr_period,
1775 1772 thisDatetime=thisDatetime,
1776 1773 update_figfile=update_figfile)
1777 1774
1778 1775 if dataOut.ltctime + dataOut.outputInterval >= self.xmax:
1779 1776 self.counter_imagwr = wr_period
1780 1777 self.isConfig = False
1781 1778 update_figfile = True
1782 1779
1783 1780
1784 1781
1785 1782 class NSMeteorDetection1Plot(Figure):
1786 1783
1787 1784 isConfig = None
1788 1785 __nsubplots = None
1789 1786
1790 1787 WIDTHPROF = None
1791 1788 HEIGHTPROF = None
1792 1789 PREFIX = 'nsm'
1793 1790
1794 1791 zminList = None
1795 1792 zmaxList = None
1796 1793 cmapList = None
1797 1794 titleList = None
1798 1795 nPairs = None
1799 1796 nChannels = None
1800 1797 nParam = None
1801 1798
1802 1799 def __init__(self, **kwargs):
1803 1800 Figure.__init__(self, **kwargs)
1804 1801 self.isConfig = False
1805 1802 self.__nsubplots = 1
1806 1803
1807 1804 self.WIDTH = 750
1808 1805 self.HEIGHT = 250
1809 1806 self.WIDTHPROF = 120
1810 1807 self.HEIGHTPROF = 0
1811 1808 self.counter_imagwr = 0
1812 1809
1813 1810 self.PLOT_CODE = SPEC_CODE
1814 1811
1815 1812 self.FTP_WEI = None
1816 1813 self.EXP_CODE = None
1817 1814 self.SUB_EXP_CODE = None
1818 1815 self.PLOT_POS = None
1819 1816
1820 1817 self.__xfilter_ena = False
1821 1818 self.__yfilter_ena = False
1822 1819
1823 1820 def getSubplots(self):
1824 1821
1825 1822 ncol = 3
1826 1823 nrow = int(numpy.ceil(self.nplots/3.0))
1827 1824
1828 1825 return nrow, ncol
1829 1826
1830 1827 def setup(self, id, nplots, wintitle, show=True):
1831 1828
1832 1829 self.nplots = nplots
1833 1830
1834 1831 ncolspan = 1
1835 1832 colspan = 1
1836 1833
1837 1834 self.createFigure(id = id,
1838 1835 wintitle = wintitle,
1839 1836 widthplot = self.WIDTH + self.WIDTHPROF,
1840 1837 heightplot = self.HEIGHT + self.HEIGHTPROF,
1841 1838 show=show)
1842 1839
1843 1840 nrow, ncol = self.getSubplots()
1844 1841
1845 1842 counter = 0
1846 1843 for y in range(nrow):
1847 1844 for x in range(ncol):
1848 1845
1849 1846 if counter >= self.nplots:
1850 1847 break
1851 1848
1852 1849 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1853 1850
1854 1851 counter += 1
1855 1852
1856 1853 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
1857 1854 xmin=None, xmax=None, ymin=None, ymax=None, SNRmin=None, SNRmax=None,
1858 1855 vmin=None, vmax=None, wmin=None, wmax=None, mode = 'SA',
1859 1856 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1860 1857 server=None, folder=None, username=None, password=None,
1861 1858 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
1862 1859 xaxis="frequency"):
1863 1860
1864 1861 """
1865 1862
1866 1863 Input:
1867 1864 dataOut :
1868 1865 id :
1869 1866 wintitle :
1870 1867 channelList :
1871 1868 showProfile :
1872 1869 xmin : None,
1873 1870 xmax : None,
1874 1871 ymin : None,
1875 1872 ymax : None,
1876 1873 zmin : None,
1877 1874 zmax : None
1878 1875 """
1879 1876 #SEPARAR EN DOS PLOTS
1880 1877 nParam = dataOut.data_param.shape[1] - 3
1881 1878
1882 1879 utctime = dataOut.data_param[0,0]
1883 1880 tmet = dataOut.data_param[:,1].astype(int)
1884 1881 hmet = dataOut.data_param[:,2].astype(int)
1885 1882
1886 1883 x = dataOut.abscissaList
1887 1884 y = dataOut.heightList
1888 1885
1889 1886 z = numpy.zeros((nParam, y.size, x.size - 1))
1890 1887 z[:,:] = numpy.nan
1891 1888 z[:,hmet,tmet] = dataOut.data_param[:,3:].T
1892 1889 z[0,:,:] = 10*numpy.log10(z[0,:,:])
1893 1890
1894 1891 xlabel = "Time (s)"
1895 1892 ylabel = "Range (km)"
1896 1893
1897 1894 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
1898 1895
1899 1896 if not self.isConfig:
1900 1897
1901 1898 nplots = nParam
1902 1899
1903 1900 self.setup(id=id,
1904 1901 nplots=nplots,
1905 1902 wintitle=wintitle,
1906 1903 show=show)
1907 1904
1908 1905 if xmin is None: xmin = numpy.nanmin(x)
1909 1906 if xmax is None: xmax = numpy.nanmax(x)
1910 1907 if ymin is None: ymin = numpy.nanmin(y)
1911 1908 if ymax is None: ymax = numpy.nanmax(y)
1912 1909 if SNRmin is None: SNRmin = numpy.nanmin(z[0,:])
1913 1910 if SNRmax is None: SNRmax = numpy.nanmax(z[0,:])
1914 1911 if vmax is None: vmax = numpy.nanmax(numpy.abs(z[1,:]))
1915 1912 if vmin is None: vmin = -vmax
1916 1913 if wmin is None: wmin = 0
1917 1914 if wmax is None: wmax = 50
1918 1915
1919 1916 pairsList = dataOut.groupList
1920 1917 self.nPairs = len(dataOut.groupList)
1921 1918
1922 1919 zminList = [SNRmin, vmin, cmin] + [pmin]*self.nPairs
1923 1920 zmaxList = [SNRmax, vmax, cmax] + [pmax]*self.nPairs
1924 1921 titleList = ["SNR","Radial Velocity","Coherence"]
1925 1922 cmapList = ["jet","RdBu_r","jet"]
1926 1923
1927 1924 for i in range(self.nPairs):
1928 1925 strAux1 = "Phase Difference "+ str(pairsList[i][0]) + str(pairsList[i][1])
1929 1926 titleList = titleList + [strAux1]
1930 1927 cmapList = cmapList + ["RdBu_r"]
1931 1928
1932 1929 self.zminList = zminList
1933 1930 self.zmaxList = zmaxList
1934 1931 self.cmapList = cmapList
1935 1932 self.titleList = titleList
1936 1933
1937 1934 self.FTP_WEI = ftp_wei
1938 1935 self.EXP_CODE = exp_code
1939 1936 self.SUB_EXP_CODE = sub_exp_code
1940 1937 self.PLOT_POS = plot_pos
1941 1938
1942 1939 self.isConfig = True
1943 1940
1944 1941 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
1945 1942
1946 1943 for i in range(nParam):
1947 1944 title = self.titleList[i] + ": " +str_datetime
1948 1945 axes = self.axesList[i]
1949 1946 axes.pcolor(x, y, z[i,:].T,
1950 1947 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=self.zminList[i], zmax=self.zmaxList[i],
1951 1948 xlabel=xlabel, ylabel=ylabel, title=title, colormap=self.cmapList[i],ticksize=9, cblabel='')
1952 1949 self.draw()
1953 1950
1954 1951 if figfile == None:
1955 1952 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
1956 1953 name = str_datetime
1957 1954 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
1958 1955 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
1959 1956 figfile = self.getFilename(name)
1960 1957
1961 1958 self.save(figpath=figpath,
1962 1959 figfile=figfile,
1963 1960 save=save,
1964 1961 ftp=ftp,
1965 1962 wr_period=wr_period,
1966 1963 thisDatetime=thisDatetime)
1967 1964
1968 1965
1969 1966 class NSMeteorDetection2Plot(Figure):
1970 1967
1971 1968 isConfig = None
1972 1969 __nsubplots = None
1973 1970
1974 1971 WIDTHPROF = None
1975 1972 HEIGHTPROF = None
1976 1973 PREFIX = 'nsm'
1977 1974
1978 1975 zminList = None
1979 1976 zmaxList = None
1980 1977 cmapList = None
1981 1978 titleList = None
1982 1979 nPairs = None
1983 1980 nChannels = None
1984 1981 nParam = None
1985 1982
1986 1983 def __init__(self, **kwargs):
1987 1984 Figure.__init__(self, **kwargs)
1988 1985 self.isConfig = False
1989 1986 self.__nsubplots = 1
1990 1987
1991 1988 self.WIDTH = 750
1992 1989 self.HEIGHT = 250
1993 1990 self.WIDTHPROF = 120
1994 1991 self.HEIGHTPROF = 0
1995 1992 self.counter_imagwr = 0
1996 1993
1997 1994 self.PLOT_CODE = SPEC_CODE
1998 1995
1999 1996 self.FTP_WEI = None
2000 1997 self.EXP_CODE = None
2001 1998 self.SUB_EXP_CODE = None
2002 1999 self.PLOT_POS = None
2003 2000
2004 2001 self.__xfilter_ena = False
2005 2002 self.__yfilter_ena = False
2006 2003
2007 2004 def getSubplots(self):
2008 2005
2009 2006 ncol = 3
2010 2007 nrow = int(numpy.ceil(self.nplots/3.0))
2011 2008
2012 2009 return nrow, ncol
2013 2010
2014 2011 def setup(self, id, nplots, wintitle, show=True):
2015 2012
2016 2013 self.nplots = nplots
2017 2014
2018 2015 ncolspan = 1
2019 2016 colspan = 1
2020 2017
2021 2018 self.createFigure(id = id,
2022 2019 wintitle = wintitle,
2023 2020 widthplot = self.WIDTH + self.WIDTHPROF,
2024 2021 heightplot = self.HEIGHT + self.HEIGHTPROF,
2025 2022 show=show)
2026 2023
2027 2024 nrow, ncol = self.getSubplots()
2028 2025
2029 2026 counter = 0
2030 2027 for y in range(nrow):
2031 2028 for x in range(ncol):
2032 2029
2033 2030 if counter >= self.nplots:
2034 2031 break
2035 2032
2036 2033 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
2037 2034
2038 2035 counter += 1
2039 2036
2040 2037 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
2041 2038 xmin=None, xmax=None, ymin=None, ymax=None, SNRmin=None, SNRmax=None,
2042 2039 vmin=None, vmax=None, wmin=None, wmax=None, mode = 'SA',
2043 2040 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
2044 2041 server=None, folder=None, username=None, password=None,
2045 2042 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
2046 2043 xaxis="frequency"):
2047 2044
2048 2045 """
2049 2046
2050 2047 Input:
2051 2048 dataOut :
2052 2049 id :
2053 2050 wintitle :
2054 2051 channelList :
2055 2052 showProfile :
2056 2053 xmin : None,
2057 2054 xmax : None,
2058 2055 ymin : None,
2059 2056 ymax : None,
2060 2057 zmin : None,
2061 2058 zmax : None
2062 2059 """
2063 2060 #Rebuild matrix
2064 2061 utctime = dataOut.data_param[0,0]
2065 2062 cmet = dataOut.data_param[:,1].astype(int)
2066 2063 tmet = dataOut.data_param[:,2].astype(int)
2067 2064 hmet = dataOut.data_param[:,3].astype(int)
2068 2065
2069 2066 nParam = 3
2070 2067 nChan = len(dataOut.groupList)
2071 2068 x = dataOut.abscissaList
2072 2069 y = dataOut.heightList
2073 2070
2074 2071 z = numpy.full((nChan, nParam, y.size, x.size - 1),numpy.nan)
2075 2072 z[cmet,:,hmet,tmet] = dataOut.data_param[:,4:]
2076 2073 z[:,0,:,:] = 10*numpy.log10(z[:,0,:,:]) #logarithmic scale
2077 2074 z = numpy.reshape(z, (nChan*nParam, y.size, x.size-1))
2078 2075
2079 2076 xlabel = "Time (s)"
2080 2077 ylabel = "Range (km)"
2081 2078
2082 2079 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
2083 2080
2084 2081 if not self.isConfig:
2085 2082
2086 2083 nplots = nParam*nChan
2087 2084
2088 2085 self.setup(id=id,
2089 2086 nplots=nplots,
2090 2087 wintitle=wintitle,
2091 2088 show=show)
2092 2089
2093 2090 if xmin is None: xmin = numpy.nanmin(x)
2094 2091 if xmax is None: xmax = numpy.nanmax(x)
2095 2092 if ymin is None: ymin = numpy.nanmin(y)
2096 2093 if ymax is None: ymax = numpy.nanmax(y)
2097 2094 if SNRmin is None: SNRmin = numpy.nanmin(z[0,:])
2098 2095 if SNRmax is None: SNRmax = numpy.nanmax(z[0,:])
2099 2096 if vmax is None: vmax = numpy.nanmax(numpy.abs(z[1,:]))
2100 2097 if vmin is None: vmin = -vmax
2101 2098 if wmin is None: wmin = 0
2102 2099 if wmax is None: wmax = 50
2103 2100
2104 2101 self.nChannels = nChan
2105 2102
2106 2103 zminList = []
2107 2104 zmaxList = []
2108 2105 titleList = []
2109 2106 cmapList = []
2110 2107 for i in range(self.nChannels):
2111 2108 strAux1 = "SNR Channel "+ str(i)
2112 2109 strAux2 = "Radial Velocity Channel "+ str(i)
2113 2110 strAux3 = "Spectral Width Channel "+ str(i)
2114 2111
2115 2112 titleList = titleList + [strAux1,strAux2,strAux3]
2116 2113 cmapList = cmapList + ["jet","RdBu_r","jet"]
2117 2114 zminList = zminList + [SNRmin,vmin,wmin]
2118 2115 zmaxList = zmaxList + [SNRmax,vmax,wmax]
2119 2116
2120 2117 self.zminList = zminList
2121 2118 self.zmaxList = zmaxList
2122 2119 self.cmapList = cmapList
2123 2120 self.titleList = titleList
2124 2121
2125 2122 self.FTP_WEI = ftp_wei
2126 2123 self.EXP_CODE = exp_code
2127 2124 self.SUB_EXP_CODE = sub_exp_code
2128 2125 self.PLOT_POS = plot_pos
2129 2126
2130 2127 self.isConfig = True
2131 2128
2132 2129 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
2133 2130
2134 2131 for i in range(self.nplots):
2135 2132 title = self.titleList[i] + ": " +str_datetime
2136 2133 axes = self.axesList[i]
2137 2134 axes.pcolor(x, y, z[i,:].T,
2138 2135 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=self.zminList[i], zmax=self.zmaxList[i],
2139 2136 xlabel=xlabel, ylabel=ylabel, title=title, colormap=self.cmapList[i],ticksize=9, cblabel='')
2140 2137 self.draw()
2141 2138
2142 2139 if figfile == None:
2143 2140 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
2144 2141 name = str_datetime
2145 2142 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
2146 2143 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
2147 2144 figfile = self.getFilename(name)
2148 2145
2149 2146 self.save(figpath=figpath,
2150 2147 figfile=figfile,
2151 2148 save=save,
2152 2149 ftp=ftp,
2153 2150 wr_period=wr_period,
2154 2151 thisDatetime=thisDatetime)
Show file before | Show file after | Hide comments
@@ -1,481 +1,481
1 1 import numpy
2 2 import datetime
3 3 import sys
4 4 import matplotlib
5 5
6 6 if 'linux' in sys.platform:
7 matplotlib.use("TKAgg")
7 matplotlib.use("GTK3Agg")
8 8
9 9 if 'darwin' in sys.platform:
10 10 matplotlib.use('TKAgg')
11 11 #Qt4Agg', 'GTK', 'GTKAgg', 'ps', 'agg', 'cairo', 'MacOSX', 'GTKCairo', 'WXAgg', 'template', 'TkAgg', 'GTK3Cairo', 'GTK3Agg', 'svg', 'WebAgg', 'CocoaAgg', 'emf', 'gdk', 'WX'
12 12 import matplotlib.pyplot
13 13
14 14 from mpl_toolkits.axes_grid1 import make_axes_locatable
15 15 from matplotlib.ticker import FuncFormatter, LinearLocator
16 16
17 17 ###########################################
18 18 #Actualizacion de las funciones del driver
19 19 ###########################################
20 20
21 21 # create jro colormap
22 22
23 23 jet_values = matplotlib.pyplot.get_cmap("jet", 100)(numpy.arange(100))[10:90]
24 24 blu_values = matplotlib.pyplot.get_cmap("seismic_r", 20)(numpy.arange(20))[10:15]
25 25 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list("jro", numpy.vstack((blu_values, jet_values)))
26 26 matplotlib.pyplot.register_cmap(cmap=ncmap)
27 27
28 28 def createFigure(id, wintitle, width, height, facecolor="w", show=True, dpi = 80):
29 29
30 30 matplotlib.pyplot.ioff()
31 31
32 32 fig = matplotlib.pyplot.figure(num=id, facecolor=facecolor, figsize=(1.0*width/dpi, 1.0*height/dpi))
33 33 fig.canvas.manager.set_window_title(wintitle)
34 34 # fig.canvas.manager.resize(width, height)
35 35 matplotlib.pyplot.ion()
36 36
37 37 if show:
38 38 matplotlib.pyplot.show()
39 39
40 40 return fig
41 41
42 42 def closeFigure(show=False, fig=None):
43 43
44 44 # matplotlib.pyplot.ioff()
45 45 # matplotlib.pyplot.pause(0)
46 46
47 47 if show:
48 48 matplotlib.pyplot.show()
49 49
50 50 if fig != None:
51 51 matplotlib.pyplot.close(fig)
52 52 # matplotlib.pyplot.pause(0)
53 53 # matplotlib.pyplot.ion()
54 54
55 55 return
56 56
57 57 matplotlib.pyplot.close("all")
58 58 # matplotlib.pyplot.pause(0)
59 59 # matplotlib.pyplot.ion()
60 60
61 61 return
62 62
63 63 def saveFigure(fig, filename):
64 64
65 65 # matplotlib.pyplot.ioff()
66 66 fig.savefig(filename, dpi=matplotlib.pyplot.gcf().dpi)
67 67 # matplotlib.pyplot.ion()
68 68
69 69 def clearFigure(fig):
70 70
71 71 fig.clf()
72 72
73 73 def setWinTitle(fig, title):
74 74
75 75 fig.canvas.manager.set_window_title(title)
76 76
77 77 def setTitle(fig, title):
78 78
79 79 fig.suptitle(title)
80 80
81 81 def createAxes(fig, nrow, ncol, xpos, ypos, colspan, rowspan, polar=False):
82 82
83 83 matplotlib.pyplot.ioff()
84 84 matplotlib.pyplot.figure(fig.number)
85 85 axes = matplotlib.pyplot.subplot2grid((nrow, ncol),
86 86 (xpos, ypos),
87 87 colspan=colspan,
88 88 rowspan=rowspan,
89 89 polar=polar)
90 90
91 91 axes.grid(True)
92 92
93 93 matplotlib.pyplot.ion()
94 94 return axes
95 95
96 96 def setAxesText(ax, text):
97 97
98 98 ax.annotate(text,
99 99 xy = (.1, .99),
100 100 xycoords = 'figure fraction',
101 101 horizontalalignment = 'left',
102 102 verticalalignment = 'top',
103 103 fontsize = 10)
104 104
105 105 def printLabels(ax, xlabel, ylabel, title):
106 106
107 107 ax.set_xlabel(xlabel, size=11)
108 108 ax.set_ylabel(ylabel, size=11)
109 109 ax.set_title(title, size=8)
110 110
111 111 def createPline(ax, x, y, xmin, xmax, ymin, ymax, xlabel='', ylabel='', title='',
112 112 ticksize=9, xtick_visible=True, ytick_visible=True,
113 113 nxticks=4, nyticks=10,
114 114 grid=None,color='blue'):
115 115
116 116 """
117 117
118 118 Input:
119 119 grid : None, 'both', 'x', 'y'
120 120 """
121 121
122 122 matplotlib.pyplot.ioff()
123 123
124 124 ax.set_xlim([xmin,xmax])
125 125 ax.set_ylim([ymin,ymax])
126 126
127 127 printLabels(ax, xlabel, ylabel, title)
128 128
129 129 ######################################################
130 130 if (xmax-xmin)<=1:
131 131 xtickspos = numpy.linspace(xmin,xmax,nxticks)
132 132 xtickspos = numpy.array([float("%.1f"%i) for i in xtickspos])
133 133 ax.set_xticks(xtickspos)
134 134 else:
135 135 xtickspos = numpy.arange(nxticks)*int((xmax-xmin)/(nxticks)) + int(xmin)
136 136 # xtickspos = numpy.arange(nxticks)*float(xmax-xmin)/float(nxticks) + int(xmin)
137 137 ax.set_xticks(xtickspos)
138 138
139 139 for tick in ax.get_xticklabels():
140 140 tick.set_visible(xtick_visible)
141 141
142 142 for tick in ax.xaxis.get_major_ticks():
143 143 tick.label.set_fontsize(ticksize)
144 144
145 145 ######################################################
146 146 for tick in ax.get_yticklabels():
147 147 tick.set_visible(ytick_visible)
148 148
149 149 for tick in ax.yaxis.get_major_ticks():
150 150 tick.label.set_fontsize(ticksize)
151 151
152 152 ax.plot(x, y, color=color)
153 153 iplot = ax.lines[-1]
154 154
155 155 ######################################################
156 156 if '0.' in matplotlib.__version__[0:2]:
157 157 print "The matplotlib version has to be updated to 1.1 or newer"
158 158 return iplot
159 159
160 160 if '1.0.' in matplotlib.__version__[0:4]:
161 161 print "The matplotlib version has to be updated to 1.1 or newer"
162 162 return iplot
163 163
164 164 if grid != None:
165 165 ax.grid(b=True, which='major', axis=grid)
166 166
167 167 matplotlib.pyplot.tight_layout()
168 168
169 169 matplotlib.pyplot.ion()
170 170
171 171 return iplot
172 172
173 173 def set_linedata(ax, x, y, idline):
174 174
175 175 ax.lines[idline].set_data(x,y)
176 176
177 177 def pline(iplot, x, y, xlabel='', ylabel='', title=''):
178 178
179 179 ax = iplot.get_axes()
180 180
181 181 printLabels(ax, xlabel, ylabel, title)
182 182
183 183 set_linedata(ax, x, y, idline=0)
184 184
185 185 def addpline(ax, x, y, color, linestyle, lw):
186 186
187 187 ax.plot(x,y,color=color,linestyle=linestyle,lw=lw)
188 188
189 189
190 190 def createPcolor(ax, x, y, z, xmin, xmax, ymin, ymax, zmin, zmax,
191 191 xlabel='', ylabel='', title='', ticksize = 9,
192 192 colormap='jet',cblabel='', cbsize="5%",
193 193 XAxisAsTime=False):
194 194
195 195 matplotlib.pyplot.ioff()
196 196
197 197 divider = make_axes_locatable(ax)
198 198 ax_cb = divider.new_horizontal(size=cbsize, pad=0.05)
199 199 fig = ax.get_figure()
200 200 fig.add_axes(ax_cb)
201 201
202 202 ax.set_xlim([xmin,xmax])
203 203 ax.set_ylim([ymin,ymax])
204 204
205 205 printLabels(ax, xlabel, ylabel, title)
206 206
207 207 z = numpy.ma.masked_invalid(z)
208 208 cmap=matplotlib.pyplot.get_cmap(colormap)
209 209 cmap.set_bad('white',1.)
210 210 imesh = ax.pcolormesh(x,y,z.T, vmin=zmin, vmax=zmax, cmap=cmap)
211 211 cb = matplotlib.pyplot.colorbar(imesh, cax=ax_cb)
212 212 cb.set_label(cblabel)
213 213
214 214 # for tl in ax_cb.get_yticklabels():
215 215 # tl.set_visible(True)
216 216
217 217 for tick in ax.yaxis.get_major_ticks():
218 218 tick.label.set_fontsize(ticksize)
219 219
220 220 for tick in ax.xaxis.get_major_ticks():
221 221 tick.label.set_fontsize(ticksize)
222 222
223 223 for tick in cb.ax.get_yticklabels():
224 224 tick.set_fontsize(ticksize)
225 225
226 226 ax_cb.yaxis.tick_right()
227 227
228 228 if '0.' in matplotlib.__version__[0:2]:
229 229 print "The matplotlib version has to be updated to 1.1 or newer"
230 230 return imesh
231 231
232 232 if '1.0.' in matplotlib.__version__[0:4]:
233 233 print "The matplotlib version has to be updated to 1.1 or newer"
234 234 return imesh
235 235
236 236 matplotlib.pyplot.tight_layout()
237 237
238 238 if XAxisAsTime:
239 239
240 240 func = lambda x, pos: ('%s') %(datetime.datetime.utcfromtimestamp(x).strftime("%H:%M:%S"))
241 241 ax.xaxis.set_major_formatter(FuncFormatter(func))
242 242 ax.xaxis.set_major_locator(LinearLocator(7))
243 243
244 244 ax.grid(True)
245 245 matplotlib.pyplot.ion()
246 246 return imesh
247 247
248 248 def pcolor(imesh, z, xlabel='', ylabel='', title=''):
249 249
250 250 z = numpy.ma.masked_invalid(z)
251 251
252 252 cmap=matplotlib.pyplot.get_cmap('jet')
253 253 cmap.set_bad('white',1.)
254 254
255 255 z = z.T
256 256 ax = imesh.get_axes()
257 257 printLabels(ax, xlabel, ylabel, title)
258 258 imesh.set_array(z.ravel())
259 259 ax.grid(True)
260 260
261 261
262 262 def addpcolor(ax, x, y, z, zmin, zmax, xlabel='', ylabel='', title='', colormap='jet'):
263 263
264 264 printLabels(ax, xlabel, ylabel, title)
265 265 z = numpy.ma.masked_invalid(z)
266 266 cmap=matplotlib.pyplot.get_cmap(colormap)
267 267 cmap.set_bad('white',1.)
268 268 ax.pcolormesh(x,y,z.T,vmin=zmin,vmax=zmax, cmap=matplotlib.pyplot.get_cmap(colormap))
269 269 ax.grid(True)
270 270
271 271 def addpcolorbuffer(ax, x, y, z, zmin, zmax, xlabel='', ylabel='', title='', colormap='jet'):
272 272
273 273 printLabels(ax, xlabel, ylabel, title)
274 274
275 275 ax.collections.remove(ax.collections[0])
276 276
277 277 z = numpy.ma.masked_invalid(z)
278 278
279 279 cmap=matplotlib.pyplot.get_cmap(colormap)
280 280 cmap.set_bad('white',1.)
281 281
282 282 ax.pcolormesh(x,y,z.T,vmin=zmin,vmax=zmax, cmap=cmap)
283 283 ax.grid(True)
284 284
285 285
286 286 def createPmultiline(ax, x, y, xmin, xmax, ymin, ymax, xlabel='', ylabel='', title='', legendlabels=None,
287 287 ticksize=9, xtick_visible=True, ytick_visible=True,
288 288 nxticks=4, nyticks=10,
289 289 grid=None):
290 290
291 291 """
292 292
293 293 Input:
294 294 grid : None, 'both', 'x', 'y'
295 295 """
296 296
297 297 matplotlib.pyplot.ioff()
298 298
299 299 lines = ax.plot(x.T, y)
300 300 leg = ax.legend(lines, legendlabels, loc='upper right')
301 301 leg.get_frame().set_alpha(0.5)
302 302 ax.set_xlim([xmin,xmax])
303 303 ax.set_ylim([ymin,ymax])
304 304 printLabels(ax, xlabel, ylabel, title)
305 305
306 306 xtickspos = numpy.arange(nxticks)*int((xmax-xmin)/(nxticks)) + int(xmin)
307 307 ax.set_xticks(xtickspos)
308 308
309 309 for tick in ax.get_xticklabels():
310 310 tick.set_visible(xtick_visible)
311 311
312 312 for tick in ax.xaxis.get_major_ticks():
313 313 tick.label.set_fontsize(ticksize)
314 314
315 315 for tick in ax.get_yticklabels():
316 316 tick.set_visible(ytick_visible)
317 317
318 318 for tick in ax.yaxis.get_major_ticks():
319 319 tick.label.set_fontsize(ticksize)
320 320
321 321 iplot = ax.lines[-1]
322 322
323 323 if '0.' in matplotlib.__version__[0:2]:
324 324 print "The matplotlib version has to be updated to 1.1 or newer"
325 325 return iplot
326 326
327 327 if '1.0.' in matplotlib.__version__[0:4]:
328 328 print "The matplotlib version has to be updated to 1.1 or newer"
329 329 return iplot
330 330
331 331 if grid != None:
332 332 ax.grid(b=True, which='major', axis=grid)
333 333
334 334 matplotlib.pyplot.tight_layout()
335 335
336 336 matplotlib.pyplot.ion()
337 337
338 338 return iplot
339 339
340 340
341 341 def pmultiline(iplot, x, y, xlabel='', ylabel='', title=''):
342 342
343 343 ax = iplot.get_axes()
344 344
345 345 printLabels(ax, xlabel, ylabel, title)
346 346
347 347 for i in range(len(ax.lines)):
348 348 line = ax.lines[i]
349 349 line.set_data(x[i,:],y)
350 350
351 351 def createPmultilineYAxis(ax, x, y, xmin, xmax, ymin, ymax, xlabel='', ylabel='', title='', legendlabels=None,
352 352 ticksize=9, xtick_visible=True, ytick_visible=True,
353 353 nxticks=4, nyticks=10, marker='.', markersize=10, linestyle="None",
354 354 grid=None, XAxisAsTime=False):
355 355
356 356 """
357 357
358 358 Input:
359 359 grid : None, 'both', 'x', 'y'
360 360 """
361 361
362 362 matplotlib.pyplot.ioff()
363 363
364 364 # lines = ax.plot(x, y.T, marker=marker,markersize=markersize,linestyle=linestyle)
365 365 lines = ax.plot(x, y.T)
366 366 # leg = ax.legend(lines, legendlabels, loc=2, bbox_to_anchor=(1.01, 1.00), numpoints=1, handlelength=1.5, \
367 367 # handletextpad=0.5, borderpad=0.5, labelspacing=0.5, borderaxespad=0.)
368 368
369 369 leg = ax.legend(lines, legendlabels,
370 370 loc='upper right', bbox_to_anchor=(1.16, 1), borderaxespad=0)
371 371
372 372 for label in leg.get_texts(): label.set_fontsize(9)
373 373
374 374 ax.set_xlim([xmin,xmax])
375 375 ax.set_ylim([ymin,ymax])
376 376 printLabels(ax, xlabel, ylabel, title)
377 377
378 378 # xtickspos = numpy.arange(nxticks)*int((xmax-xmin)/(nxticks)) + int(xmin)
379 379 # ax.set_xticks(xtickspos)
380 380
381 381 for tick in ax.get_xticklabels():
382 382 tick.set_visible(xtick_visible)
383 383
384 384 for tick in ax.xaxis.get_major_ticks():
385 385 tick.label.set_fontsize(ticksize)
386 386
387 387 for tick in ax.get_yticklabels():
388 388 tick.set_visible(ytick_visible)
389 389
390 390 for tick in ax.yaxis.get_major_ticks():
391 391 tick.label.set_fontsize(ticksize)
392 392
393 393 iplot = ax.lines[-1]
394 394
395 395 if '0.' in matplotlib.__version__[0:2]:
396 396 print "The matplotlib version has to be updated to 1.1 or newer"
397 397 return iplot
398 398
399 399 if '1.0.' in matplotlib.__version__[0:4]:
400 400 print "The matplotlib version has to be updated to 1.1 or newer"
401 401 return iplot
402 402
403 403 if grid != None:
404 404 ax.grid(b=True, which='major', axis=grid)
405 405
406 406 matplotlib.pyplot.tight_layout()
407 407
408 408 if XAxisAsTime:
409 409
410 410 func = lambda x, pos: ('%s') %(datetime.datetime.utcfromtimestamp(x).strftime("%H:%M:%S"))
411 411 ax.xaxis.set_major_formatter(FuncFormatter(func))
412 412 ax.xaxis.set_major_locator(LinearLocator(7))
413 413
414 414 matplotlib.pyplot.ion()
415 415
416 416 return iplot
417 417
418 418 def pmultilineyaxis(iplot, x, y, xlabel='', ylabel='', title=''):
419 419
420 420 ax = iplot.get_axes()
421 421 printLabels(ax, xlabel, ylabel, title)
422 422
423 423 for i in range(len(ax.lines)):
424 424 line = ax.lines[i]
425 425 line.set_data(x,y[i,:])
426 426
427 427 def createPolar(ax, x, y,
428 428 xlabel='', ylabel='', title='', ticksize = 9,
429 429 colormap='jet',cblabel='', cbsize="5%",
430 430 XAxisAsTime=False):
431 431
432 432 matplotlib.pyplot.ioff()
433 433
434 434 ax.plot(x,y,'bo', markersize=5)
435 435 # ax.set_rmax(90)
436 436 ax.set_ylim(0,90)
437 437 ax.set_yticks(numpy.arange(0,90,20))
438 438 # ax.text(0, -110, ylabel, rotation='vertical', va ='center', ha = 'center' ,size='11')
439 439 # ax.text(0, 50, ylabel, rotation='vertical', va ='center', ha = 'left' ,size='11')
440 440 # ax.text(100, 100, 'example', ha='left', va='center', rotation='vertical')
441 441 ax.yaxis.labelpad = 230
442 442 printLabels(ax, xlabel, ylabel, title)
443 443 iplot = ax.lines[-1]
444 444
445 445 if '0.' in matplotlib.__version__[0:2]:
446 446 print "The matplotlib version has to be updated to 1.1 or newer"
447 447 return iplot
448 448
449 449 if '1.0.' in matplotlib.__version__[0:4]:
450 450 print "The matplotlib version has to be updated to 1.1 or newer"
451 451 return iplot
452 452
453 453 # if grid != None:
454 454 # ax.grid(b=True, which='major', axis=grid)
455 455
456 456 matplotlib.pyplot.tight_layout()
457 457
458 458 matplotlib.pyplot.ion()
459 459
460 460
461 461 return iplot
462 462
463 463 def polar(iplot, x, y, xlabel='', ylabel='', title=''):
464 464
465 465 ax = iplot.get_axes()
466 466
467 467 # ax.text(0, -110, ylabel, rotation='vertical', va ='center', ha = 'center',size='11')
468 468 printLabels(ax, xlabel, ylabel, title)
469 469
470 470 set_linedata(ax, x, y, idline=0)
471 471
472 472 def draw(fig):
473 473
474 474 if type(fig) == 'int':
475 475 raise ValueError, "Error drawing: Fig parameter should be a matplotlib figure object figure"
476 476
477 477 fig.canvas.draw()
478 478
479 479 def pause(interval=0.000001):
480 480
481 481 matplotlib.pyplot.pause(interval)
Show file before | Show file after | Hide comments
@@ -1,21 +1,20
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $
5 5 '''
6 6
7 7 from jroIO_voltage import *
8 8 from jroIO_spectra import *
9 9 from jroIO_heispectra import *
10 10 from jroIO_usrp import *
11 11
12 12 from jroIO_kamisr import *
13 13 from jroIO_param import *
14 14 from jroIO_hf import *
15 15
16 16 from jroIO_madrigal import *
17 17
18 18 from bltrIO_param import *
19 19 from jroIO_bltr import *
20 20 from jroIO_mira35c import *
21
Show file before | Show file after | Hide comments
@@ -1,1795 +1,1807
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time, datetime
14 14 import traceback
15 15 import zmq
16 16
17 17 try:
18 18 from gevent import sleep
19 19 except:
20 20 from time import sleep
21 21
22 22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24 24
25 25 LOCALTIME = True
26 26
27 27 def isNumber(cad):
28 28 """
29 29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
30 30
31 31 Excepciones:
32 32 Si un determinado string no puede ser convertido a numero
33 33 Input:
34 34 str, string al cual se le analiza para determinar si convertible a un numero o no
35 35
36 36 Return:
37 37 True : si el string es uno numerico
38 38 False : no es un string numerico
39 39 """
40 40 try:
41 41 float( cad )
42 42 return True
43 43 except:
44 44 return False
45 45
46 46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
47 47 """
48 48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
49 49
50 50 Inputs:
51 51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
52 52
53 53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
54 54 segundos contados desde 01/01/1970.
55 55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
56 56 segundos contados desde 01/01/1970.
57 57
58 58 Return:
59 59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
60 60 fecha especificado, de lo contrario retorna False.
61 61
62 62 Excepciones:
63 63 Si el archivo no existe o no puede ser abierto
64 64 Si la cabecera no puede ser leida.
65 65
66 66 """
67 67 basicHeaderObj = BasicHeader(LOCALTIME)
68 68
69 69 try:
70 70 fp = open(filename,'rb')
71 71 except IOError:
72 72 print "The file %s can't be opened" %(filename)
73 73 return 0
74 74
75 75 sts = basicHeaderObj.read(fp)
76 76 fp.close()
77 77
78 78 if not(sts):
79 79 print "Skipping the file %s because it has not a valid header" %(filename)
80 80 return 0
81 81
82 82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
83 83 return 0
84 84
85 85 return 1
86 86
87 87 def isTimeInRange(thisTime, startTime, endTime):
88 88
89 89 if endTime >= startTime:
90 90 if (thisTime < startTime) or (thisTime > endTime):
91 91 return 0
92 92
93 93 return 1
94 94 else:
95 95 if (thisTime < startTime) and (thisTime > endTime):
96 96 return 0
97 97
98 98 return 1
99 99
100 100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
101 101 """
102 102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
103 103
104 104 Inputs:
105 105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
106 106
107 107 startDate : fecha inicial del rango seleccionado en formato datetime.date
108 108
109 109 endDate : fecha final del rango seleccionado en formato datetime.date
110 110
111 111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
112 112
113 113 endTime : tiempo final del rango seleccionado en formato datetime.time
114 114
115 115 Return:
116 116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
117 117 fecha especificado, de lo contrario retorna False.
118 118
119 119 Excepciones:
120 120 Si el archivo no existe o no puede ser abierto
121 121 Si la cabecera no puede ser leida.
122 122
123 123 """
124 124
125 125
126 126 try:
127 127 fp = open(filename,'rb')
128 128 except IOError:
129 129 print "The file %s can't be opened" %(filename)
130 130 return None
131 131
132 132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
133 133 systemHeaderObj = SystemHeader()
134 134 radarControllerHeaderObj = RadarControllerHeader()
135 135 processingHeaderObj = ProcessingHeader()
136 136
137 137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
138 138
139 139 sts = firstBasicHeaderObj.read(fp)
140 140
141 141 if not(sts):
142 142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
143 143 return None
144 144
145 145 if not systemHeaderObj.read(fp):
146 146 return None
147 147
148 148 if not radarControllerHeaderObj.read(fp):
149 149 return None
150 150
151 151 if not processingHeaderObj.read(fp):
152 152 return None
153 153
154 154 filesize = os.path.getsize(filename)
155 155
156 156 offset = processingHeaderObj.blockSize + 24 #header size
157 157
158 158 if filesize <= offset:
159 159 print "[Reading] %s: This file has not enough data" %filename
160 160 return None
161 161
162 162 fp.seek(-offset, 2)
163 163
164 164 sts = lastBasicHeaderObj.read(fp)
165 165
166 166 fp.close()
167 167
168 168 thisDatetime = lastBasicHeaderObj.datatime
169 169 thisTime_last_block = thisDatetime.time()
170 170
171 171 thisDatetime = firstBasicHeaderObj.datatime
172 172 thisDate = thisDatetime.date()
173 173 thisTime_first_block = thisDatetime.time()
174 174
175 175 #General case
176 176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
177 177 #-----------o----------------------------o-----------
178 178 # startTime endTime
179 179
180 180 if endTime >= startTime:
181 181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
182 182 return None
183 183
184 184 return thisDatetime
185 185
186 186 #If endTime < startTime then endTime belongs to the next day
187 187
188 188
189 189 #<<<<<<<<<<<o o>>>>>>>>>>>
190 190 #-----------o----------------------------o-----------
191 191 # endTime startTime
192 192
193 193 if (thisDate == startDate) and (thisTime_last_block < startTime):
194 194 return None
195 195
196 196 if (thisDate == endDate) and (thisTime_first_block > endTime):
197 197 return None
198 198
199 199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
200 200 return None
201 201
202 202 return thisDatetime
203 203
204 204 def isFolderInDateRange(folder, startDate=None, endDate=None):
205 205 """
206 206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
207 207
208 208 Inputs:
209 209 folder : nombre completo del directorio.
210 210 Su formato deberia ser "/path_root/?YYYYDDD"
211 211
212 212 siendo:
213 213 YYYY : Anio (ejemplo 2015)
214 214 DDD : Dia del anio (ejemplo 305)
215 215
216 216 startDate : fecha inicial del rango seleccionado en formato datetime.date
217 217
218 218 endDate : fecha final del rango seleccionado en formato datetime.date
219 219
220 220 Return:
221 221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
222 222 fecha especificado, de lo contrario retorna False.
223 223 Excepciones:
224 224 Si el directorio no tiene el formato adecuado
225 225 """
226 226
227 227 basename = os.path.basename(folder)
228 228
229 229 if not isRadarFolder(basename):
230 230 print "The folder %s has not the rigth format" %folder
231 231 return 0
232 232
233 233 if startDate and endDate:
234 234 thisDate = getDateFromRadarFolder(basename)
235 235
236 236 if thisDate < startDate:
237 237 return 0
238 238
239 239 if thisDate > endDate:
240 240 return 0
241 241
242 242 return 1
243 243
244 244 def isFileInDateRange(filename, startDate=None, endDate=None):
245 245 """
246 246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
247 247
248 248 Inputs:
249 249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
250 250
251 251 Su formato deberia ser "?YYYYDDDsss"
252 252
253 253 siendo:
254 254 YYYY : Anio (ejemplo 2015)
255 255 DDD : Dia del anio (ejemplo 305)
256 256 sss : set
257 257
258 258 startDate : fecha inicial del rango seleccionado en formato datetime.date
259 259
260 260 endDate : fecha final del rango seleccionado en formato datetime.date
261 261
262 262 Return:
263 263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
264 264 fecha especificado, de lo contrario retorna False.
265 265 Excepciones:
266 266 Si el archivo no tiene el formato adecuado
267 267 """
268 268
269 269 basename = os.path.basename(filename)
270 270
271 271 if not isRadarFile(basename):
272 272 print "The filename %s has not the rigth format" %filename
273 273 return 0
274 274
275 275 if startDate and endDate:
276 276 thisDate = getDateFromRadarFile(basename)
277 277
278 278 if thisDate < startDate:
279 279 return 0
280 280
281 281 if thisDate > endDate:
282 282 return 0
283 283
284 284 return 1
285 285
286 286 def getFileFromSet(path, ext, set):
287 287 validFilelist = []
288 288 fileList = os.listdir(path)
289 289
290 290 # 0 1234 567 89A BCDE
291 291 # H YYYY DDD SSS .ext
292 292
293 293 for thisFile in fileList:
294 294 try:
295 295 year = int(thisFile[1:5])
296 296 doy = int(thisFile[5:8])
297 297 except:
298 298 continue
299 299
300 300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
301 301 continue
302 302
303 303 validFilelist.append(thisFile)
304 304
305 305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
306 306
307 307 if len(myfile)!= 0:
308 308 return myfile[0]
309 309 else:
310 310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
311 311 print 'the filename %s does not exist'%filename
312 312 print '...going to the last file: '
313 313
314 314 if validFilelist:
315 315 validFilelist = sorted( validFilelist, key=str.lower )
316 316 return validFilelist[-1]
317 317
318 318 return None
319 319
320 320 def getlastFileFromPath(path, ext):
321 321 """
322 322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
323 323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
324 324
325 325 Input:
326 326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
327 327 ext : extension de los files contenidos en una carpeta
328 328
329 329 Return:
330 330 El ultimo file de una determinada carpeta, no se considera el path.
331 331 """
332 332 validFilelist = []
333 333 fileList = os.listdir(path)
334 334
335 335 # 0 1234 567 89A BCDE
336 336 # H YYYY DDD SSS .ext
337 337
338 338 for thisFile in fileList:
339 339
340 340 year = thisFile[1:5]
341 341 if not isNumber(year):
342 342 continue
343 343
344 344 doy = thisFile[5:8]
345 345 if not isNumber(doy):
346 346 continue
347 347
348 348 year = int(year)
349 349 doy = int(doy)
350 350
351 351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
352 352 continue
353 353
354 354 validFilelist.append(thisFile)
355 355
356 356 if validFilelist:
357 357 validFilelist = sorted( validFilelist, key=str.lower )
358 358 return validFilelist[-1]
359 359
360 360 return None
361 361
362 362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
363 363 """
364 364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
365 365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
366 366 el path exacto de un determinado file.
367 367
368 368 Example :
369 369 nombre correcto del file es .../.../D2009307/P2009307367.ext
370 370
371 371 Entonces la funcion prueba con las siguientes combinaciones
372 372 .../.../y2009307367.ext
373 373 .../.../Y2009307367.ext
374 374 .../.../x2009307/y2009307367.ext
375 375 .../.../x2009307/Y2009307367.ext
376 376 .../.../X2009307/y2009307367.ext
377 377 .../.../X2009307/Y2009307367.ext
378 378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
379 379
380 380 Return:
381 381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
382 382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
383 383 para el filename
384 384 """
385 385 fullfilename = None
386 386 find_flag = False
387 387 filename = None
388 388
389 389 prefixDirList = [None,'d','D']
390 390 if ext.lower() == ".r": #voltage
391 391 prefixFileList = ['d','D']
392 392 elif ext.lower() == ".pdata": #spectra
393 393 prefixFileList = ['p','P']
394 394 else:
395 395 return None, filename
396 396
397 397 #barrido por las combinaciones posibles
398 398 for prefixDir in prefixDirList:
399 399 thispath = path
400 400 if prefixDir != None:
401 401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
402 402 if foldercounter == 0:
403 403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
404 404 else:
405 405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
406 406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
407 407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
408 408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
409 409
410 410 if os.path.exists( fullfilename ): #verifico que exista
411 411 find_flag = True
412 412 break
413 413 if find_flag:
414 414 break
415 415
416 416 if not(find_flag):
417 417 return None, filename
418 418
419 419 return fullfilename, filename
420 420
421 421 def isRadarFolder(folder):
422 422 try:
423 423 year = int(folder[1:5])
424 424 doy = int(folder[5:8])
425 425 except:
426 426 return 0
427 427
428 428 return 1
429 429
430 430 def isRadarFile(file):
431 431 try:
432 432 year = int(file[1:5])
433 433 doy = int(file[5:8])
434 434 set = int(file[8:11])
435 435 except:
436 436 return 0
437 437
438 438 return 1
439 439
440 440 def getDateFromRadarFile(file):
441 441 try:
442 442 year = int(file[1:5])
443 443 doy = int(file[5:8])
444 444 set = int(file[8:11])
445 445 except:
446 446 return None
447 447
448 448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
449 449 return thisDate
450 450
451 451 def getDateFromRadarFolder(folder):
452 452 try:
453 453 year = int(folder[1:5])
454 454 doy = int(folder[5:8])
455 455 except:
456 456 return None
457 457
458 458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
459 459 return thisDate
460 460
461 461 class JRODataIO:
462 462
463 463 c = 3E8
464 464
465 465 isConfig = False
466 466
467 467 basicHeaderObj = None
468 468
469 469 systemHeaderObj = None
470 470
471 471 radarControllerHeaderObj = None
472 472
473 473 processingHeaderObj = None
474 474
475 475 dtype = None
476 476
477 477 pathList = []
478 478
479 479 filenameList = []
480 480
481 481 filename = None
482 482
483 483 ext = None
484 484
485 485 flagIsNewFile = 1
486 486
487 487 flagDiscontinuousBlock = 0
488 488
489 489 flagIsNewBlock = 0
490 490
491 491 fp = None
492 492
493 493 firstHeaderSize = 0
494 494
495 495 basicHeaderSize = 24
496 496
497 497 versionFile = 1103
498 498
499 499 fileSize = None
500 500
501 501 # ippSeconds = None
502 502
503 503 fileSizeByHeader = None
504 504
505 505 fileIndex = None
506 506
507 507 profileIndex = None
508 508
509 509 blockIndex = None
510 510
511 511 nTotalBlocks = None
512 512
513 513 maxTimeStep = 30
514 514
515 515 lastUTTime = None
516 516
517 517 datablock = None
518 518
519 519 dataOut = None
520 520
521 521 blocksize = None
522 522
523 523 getByBlock = False
524 524
525 525 def __init__(self):
526 526
527 527 raise NotImplementedError
528 528
529 529 def run(self):
530 530
531 531 raise NotImplementedError
532 532
533 533 def getDtypeWidth(self):
534 534
535 535 dtype_index = get_dtype_index(self.dtype)
536 536 dtype_width = get_dtype_width(dtype_index)
537 537
538 538 return dtype_width
539 539
540 540 def getAllowedArgs(self):
541 541 return inspect.getargspec(self.run).args
542 542
543 543 class JRODataReader(JRODataIO):
544 544
545 545 online = 0
546 546
547 547 realtime = 0
548 548
549 549 nReadBlocks = 0
550 550
551 551 delay = 10 #number of seconds waiting a new file
552 552
553 553 nTries = 3 #quantity tries
554 554
555 555 nFiles = 3 #number of files for searching
556 556
557 557 path = None
558 558
559 559 foldercounter = 0
560 560
561 561 flagNoMoreFiles = 0
562 562
563 563 datetimeList = []
564 564
565 565 __isFirstTimeOnline = 1
566 566
567 567 __printInfo = True
568 568
569 569 profileIndex = None
570 570
571 571 nTxs = 1
572 572
573 573 txIndex = None
574 574
575 575 #Added--------------------
576 576
577 577 selBlocksize = None
578 578
579 579 selBlocktime = None
580 580
581 581 def __init__(self):
582 582
583 583 """
584 584 This class is used to find data files
585 585
586 586 Example:
587 587 reader = JRODataReader()
588 588 fileList = reader.findDataFiles()
589 589
590 590 """
591 591 pass
592 592
593 593
594 594 def createObjByDefault(self):
595 595 """
596 596
597 597 """
598 598 raise NotImplementedError
599 599
600 600 def getBlockDimension(self):
601 601
602 602 raise NotImplementedError
603 603
604 604 def searchFilesOffLine(self,
605 605 path,
606 606 startDate=None,
607 607 endDate=None,
608 608 startTime=datetime.time(0,0,0),
609 609 endTime=datetime.time(23,59,59),
610 610 set=None,
611 611 expLabel='',
612 612 ext='.r',
613 613 cursor=None,
614 614 skip=None,
615 615 walk=True):
616 616
617 617 self.filenameList = []
618 618 self.datetimeList = []
619 619
620 620 pathList = []
621 621
622 622 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
623 623
624 624 if dateList == []:
625 625 return [], []
626 626
627 627 if len(dateList) > 1:
628 628 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
629 629 else:
630 630 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
631 631
632 632 filenameList = []
633 633 datetimeList = []
634 634
635 635 for thisPath in pathList:
636 636
637 637 fileList = glob.glob1(thisPath, "*%s" %ext)
638 638 fileList.sort()
639 639
640 640 skippedFileList = []
641 641
642 642 if cursor is not None and skip is not None:
643 643
644 644 if skip == 0:
645 645 skippedFileList = []
646 646 else:
647 647 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
648 648
649 649 else:
650 650 skippedFileList = fileList
651 651
652 652 for file in skippedFileList:
653 653
654 654 filename = os.path.join(thisPath,file)
655 655
656 656 if not isFileInDateRange(filename, startDate, endDate):
657 657 continue
658 658
659 659 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
660 660
661 661 if not(thisDatetime):
662 662 continue
663 663
664 664 filenameList.append(filename)
665 665 datetimeList.append(thisDatetime)
666 666
667 667 if not(filenameList):
668 668 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
669 669 return [], []
670 670
671 671 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
672 672 print
673 673
674 674 # for i in range(len(filenameList)):
675 675 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
676 676
677 677 self.filenameList = filenameList
678 678 self.datetimeList = datetimeList
679 679
680 680 return pathList, filenameList
681 681
682 682 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
683 683
684 684 """
685 685 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
686 686 devuelve el archivo encontrado ademas de otros datos.
687 687
688 688 Input:
689 689 path : carpeta donde estan contenidos los files que contiene data
690 690
691 691 expLabel : Nombre del subexperimento (subfolder)
692 692
693 693 ext : extension de los files
694 694
695 695 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
696 696
697 697 Return:
698 698 directory : eL directorio donde esta el file encontrado
699 699 filename : el ultimo file de una determinada carpeta
700 700 year : el anho
701 701 doy : el numero de dia del anho
702 702 set : el set del archivo
703 703
704 704
705 705 """
706 706 if not os.path.isdir(path):
707 707 return None, None, None, None, None, None
708 708
709 709 dirList = []
710 710
711 711 if not walk:
712 712 fullpath = path
713 713 foldercounter = 0
714 714 else:
715 715 #Filtra solo los directorios
716 716 for thisPath in os.listdir(path):
717 717 if not os.path.isdir(os.path.join(path,thisPath)):
718 718 continue
719 719 if not isRadarFolder(thisPath):
720 720 continue
721 721
722 722 dirList.append(thisPath)
723 723
724 724 if not(dirList):
725 725 return None, None, None, None, None, None
726 726
727 727 dirList = sorted( dirList, key=str.lower )
728 728
729 729 doypath = dirList[-1]
730 730 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
731 731 fullpath = os.path.join(path, doypath, expLabel)
732 732
733 733
734 734 print "[Reading] %s folder was found: " %(fullpath )
735 735
736 736 if set == None:
737 737 filename = getlastFileFromPath(fullpath, ext)
738 738 else:
739 739 filename = getFileFromSet(fullpath, ext, set)
740 740
741 741 if not(filename):
742 742 return None, None, None, None, None, None
743 743
744 744 print "[Reading] %s file was found" %(filename)
745 745
746 746 if not(self.__verifyFile(os.path.join(fullpath, filename))):
747 747 return None, None, None, None, None, None
748 748
749 749 year = int( filename[1:5] )
750 750 doy = int( filename[5:8] )
751 751 set = int( filename[8:11] )
752 752
753 753 return fullpath, foldercounter, filename, year, doy, set
754 754
755 755 def __setNextFileOffline(self):
756 756
757 757 idFile = self.fileIndex
758 758
759 759 while (True):
760 760 idFile += 1
761 761 if not(idFile < len(self.filenameList)):
762 762 self.flagNoMoreFiles = 1
763 763 # print "[Reading] No more Files"
764 764 return 0
765 765
766 766 filename = self.filenameList[idFile]
767 767
768 768 if not(self.__verifyFile(filename)):
769 769 continue
770 770
771 771 fileSize = os.path.getsize(filename)
772 772 fp = open(filename,'rb')
773 773 break
774 774
775 775 self.flagIsNewFile = 1
776 776 self.fileIndex = idFile
777 777 self.filename = filename
778 778 self.fileSize = fileSize
779 779 self.fp = fp
780 780
781 781 # print "[Reading] Setting the file: %s"%self.filename
782 782
783 783 return 1
784 784
785 785 def __setNextFileOnline(self):
786 786 """
787 787 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
788 788 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
789 789 siguientes.
790 790
791 791 Affected:
792 792 self.flagIsNewFile
793 793 self.filename
794 794 self.fileSize
795 795 self.fp
796 796 self.set
797 797 self.flagNoMoreFiles
798 798
799 799 Return:
800 800 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
801 801 1 : si el file fue abierto con exito y esta listo a ser leido
802 802
803 803 Excepciones:
804 804 Si un determinado file no puede ser abierto
805 805 """
806 806 nFiles = 0
807 807 fileOk_flag = False
808 808 firstTime_flag = True
809 809
810 810 self.set += 1
811 811
812 812 if self.set > 999:
813 813 self.set = 0
814 814 self.foldercounter += 1
815 815
816 816 #busca el 1er file disponible
817 817 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
818 818 if fullfilename:
819 819 if self.__verifyFile(fullfilename, False):
820 820 fileOk_flag = True
821 821
822 822 #si no encuentra un file entonces espera y vuelve a buscar
823 823 if not(fileOk_flag):
824 824 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
825 825
826 826 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
827 827 tries = self.nTries
828 828 else:
829 829 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
830 830
831 831 for nTries in range( tries ):
832 832 if firstTime_flag:
833 833 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
834 834 sleep( self.delay )
835 835 else:
836 836 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
837 837
838 838 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
839 839 if fullfilename:
840 840 if self.__verifyFile(fullfilename):
841 841 fileOk_flag = True
842 842 break
843 843
844 844 if fileOk_flag:
845 845 break
846 846
847 847 firstTime_flag = False
848 848
849 849 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
850 850 self.set += 1
851 851
852 852 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
853 853 self.set = 0
854 854 self.doy += 1
855 855 self.foldercounter = 0
856 856
857 857 if fileOk_flag:
858 858 self.fileSize = os.path.getsize( fullfilename )
859 859 self.filename = fullfilename
860 860 self.flagIsNewFile = 1
861 861 if self.fp != None: self.fp.close()
862 862 self.fp = open(fullfilename, 'rb')
863 863 self.flagNoMoreFiles = 0
864 864 # print '[Reading] Setting the file: %s' % fullfilename
865 865 else:
866 866 self.fileSize = 0
867 867 self.filename = None
868 868 self.flagIsNewFile = 0
869 869 self.fp = None
870 870 self.flagNoMoreFiles = 1
871 871 # print '[Reading] No more files to read'
872 872
873 873 return fileOk_flag
874 874
875 875 def setNextFile(self):
876 876 if self.fp != None:
877 877 self.fp.close()
878 878
879 879 if self.online:
880 880 newFile = self.__setNextFileOnline()
881 881 else:
882 882 newFile = self.__setNextFileOffline()
883 883
884 884 if not(newFile):
885 885 print '[Reading] No more files to read'
886 886 return 0
887 887
888 888 if self.verbose:
889 889 print '[Reading] Setting the file: %s' % self.filename
890 890
891 891 self.__readFirstHeader()
892 892 self.nReadBlocks = 0
893 893 return 1
894 894
895 895 def __waitNewBlock(self):
896 896 """
897 897 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
898 898
899 899 Si el modo de lectura es OffLine siempre retorn 0
900 900 """
901 901 if not self.online:
902 902 return 0
903 903
904 904 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
905 905 return 0
906 906
907 907 currentPointer = self.fp.tell()
908 908
909 909 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
910 910
911 911 for nTries in range( self.nTries ):
912 912
913 913 self.fp.close()
914 914 self.fp = open( self.filename, 'rb' )
915 915 self.fp.seek( currentPointer )
916 916
917 917 self.fileSize = os.path.getsize( self.filename )
918 918 currentSize = self.fileSize - currentPointer
919 919
920 920 if ( currentSize >= neededSize ):
921 921 self.basicHeaderObj.read(self.fp)
922 922 return 1
923 923
924 924 if self.fileSize == self.fileSizeByHeader:
925 925 # self.flagEoF = True
926 926 return 0
927 927
928 928 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
929 929 sleep( self.delay )
930 930
931 931
932 932 return 0
933 933
934 934 def waitDataBlock(self,pointer_location):
935 935
936 936 currentPointer = pointer_location
937 937
938 938 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
939 939
940 940 for nTries in range( self.nTries ):
941 941 self.fp.close()
942 942 self.fp = open( self.filename, 'rb' )
943 943 self.fp.seek( currentPointer )
944 944
945 945 self.fileSize = os.path.getsize( self.filename )
946 946 currentSize = self.fileSize - currentPointer
947 947
948 948 if ( currentSize >= neededSize ):
949 949 return 1
950 950
951 951 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
952 952 sleep( self.delay )
953 953
954 954 return 0
955 955
956 956 def __jumpToLastBlock(self):
957 957
958 958 if not(self.__isFirstTimeOnline):
959 959 return
960 960
961 961 csize = self.fileSize - self.fp.tell()
962 962 blocksize = self.processingHeaderObj.blockSize
963 963
964 964 #salta el primer bloque de datos
965 965 if csize > self.processingHeaderObj.blockSize:
966 966 self.fp.seek(self.fp.tell() + blocksize)
967 967 else:
968 968 return
969 969
970 970 csize = self.fileSize - self.fp.tell()
971 971 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
972 972 while True:
973 973
974 974 if self.fp.tell()<self.fileSize:
975 975 self.fp.seek(self.fp.tell() + neededsize)
976 976 else:
977 977 self.fp.seek(self.fp.tell() - neededsize)
978 978 break
979 979
980 980 # csize = self.fileSize - self.fp.tell()
981 981 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
982 982 # factor = int(csize/neededsize)
983 983 # if factor > 0:
984 984 # self.fp.seek(self.fp.tell() + factor*neededsize)
985 985
986 986 self.flagIsNewFile = 0
987 987 self.__isFirstTimeOnline = 0
988 988
989 989 def __setNewBlock(self):
990 990 #if self.server is None:
991 991 if self.fp == None:
992 992 return 0
993 993
994 994 # if self.online:
995 995 # self.__jumpToLastBlock()
996 996
997 997 if self.flagIsNewFile:
998 998 self.lastUTTime = self.basicHeaderObj.utc
999 999 return 1
1000 1000
1001 1001 if self.realtime:
1002 1002 self.flagDiscontinuousBlock = 1
1003 1003 if not(self.setNextFile()):
1004 1004 return 0
1005 1005 else:
1006 1006 return 1
1007 1007 #if self.server is None:
1008 1008 currentSize = self.fileSize - self.fp.tell()
1009 1009 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1010 1010 if (currentSize >= neededSize):
1011 1011 self.basicHeaderObj.read(self.fp)
1012 1012 self.lastUTTime = self.basicHeaderObj.utc
1013 1013 return 1
1014 1014 # else:
1015 1015 # self.basicHeaderObj.read(self.zHeader)
1016 1016 # self.lastUTTime = self.basicHeaderObj.utc
1017 1017 # return 1
1018 1018 if self.__waitNewBlock():
1019 1019 self.lastUTTime = self.basicHeaderObj.utc
1020 1020 return 1
1021 1021 #if self.server is None:
1022 1022 if not(self.setNextFile()):
1023 1023 return 0
1024 1024
1025 1025 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1026 1026 self.lastUTTime = self.basicHeaderObj.utc
1027 1027
1028 1028 self.flagDiscontinuousBlock = 0
1029 1029
1030 1030 if deltaTime > self.maxTimeStep:
1031 1031 self.flagDiscontinuousBlock = 1
1032 1032
1033 1033 return 1
1034 1034
1035 1035 def readNextBlock(self):
1036 1036
1037 1037 #Skip block out of startTime and endTime
1038 1038 while True:
1039 1039 if not(self.__setNewBlock()):
1040 1040 return 0
1041 1041
1042 1042 if not(self.readBlock()):
1043 1043 return 0
1044 1044
1045 1045 self.getBasicHeader()
1046 1046
1047 1047 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1048 1048
1049 1049 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1050 1050 self.processingHeaderObj.dataBlocksPerFile,
1051 1051 self.dataOut.datatime.ctime())
1052 1052 continue
1053 1053
1054 1054 break
1055 1055
1056 1056 if self.verbose:
1057 1057 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1058 1058 self.processingHeaderObj.dataBlocksPerFile,
1059 1059 self.dataOut.datatime.ctime())
1060 1060 return 1
1061 1061
1062 1062 def __readFirstHeader(self):
1063 1063
1064 1064 self.basicHeaderObj.read(self.fp)
1065 1065 self.systemHeaderObj.read(self.fp)
1066 1066 self.radarControllerHeaderObj.read(self.fp)
1067 1067 self.processingHeaderObj.read(self.fp)
1068 1068
1069 1069 self.firstHeaderSize = self.basicHeaderObj.size
1070 1070
1071 1071 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1072 1072 if datatype == 0:
1073 1073 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1074 1074 elif datatype == 1:
1075 1075 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1076 1076 elif datatype == 2:
1077 1077 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1078 1078 elif datatype == 3:
1079 1079 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1080 1080 elif datatype == 4:
1081 1081 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1082 1082 elif datatype == 5:
1083 1083 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1084 1084 else:
1085 1085 raise ValueError, 'Data type was not defined'
1086 1086
1087 1087 self.dtype = datatype_str
1088 1088 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1089 1089 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1090 1090 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1091 1091 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1092 1092 self.getBlockDimension()
1093 1093
1094 1094 def __verifyFile(self, filename, msgFlag=True):
1095 1095
1096 1096 msg = None
1097 1097
1098 1098 try:
1099 1099 fp = open(filename, 'rb')
1100 1100 except IOError:
1101 1101
1102 1102 if msgFlag:
1103 1103 print "[Reading] File %s can't be opened" % (filename)
1104 1104
1105 1105 return False
1106 1106
1107 1107 currentPosition = fp.tell()
1108 1108 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1109 1109
1110 1110 if neededSize == 0:
1111 1111 basicHeaderObj = BasicHeader(LOCALTIME)
1112 1112 systemHeaderObj = SystemHeader()
1113 1113 radarControllerHeaderObj = RadarControllerHeader()
1114 1114 processingHeaderObj = ProcessingHeader()
1115 1115
1116 1116 if not( basicHeaderObj.read(fp) ):
1117 1117 fp.close()
1118 1118 return False
1119 1119
1120 1120 if not( systemHeaderObj.read(fp) ):
1121 1121 fp.close()
1122 1122 return False
1123 1123
1124 1124 if not( radarControllerHeaderObj.read(fp) ):
1125 1125 fp.close()
1126 1126 return False
1127 1127
1128 1128 if not( processingHeaderObj.read(fp) ):
1129 1129 fp.close()
1130 1130 return False
1131 1131
1132 1132 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1133 1133 else:
1134 1134 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1135 1135
1136 1136 fp.close()
1137 1137
1138 1138 fileSize = os.path.getsize(filename)
1139 1139 currentSize = fileSize - currentPosition
1140 1140
1141 1141 if currentSize < neededSize:
1142 1142 if msgFlag and (msg != None):
1143 1143 print msg
1144 1144 return False
1145 1145
1146 1146 return True
1147 1147
1148 1148 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1149 1149
1150 1150 path_empty = True
1151 1151
1152 1152 dateList = []
1153 1153 pathList = []
1154 1154
1155 1155 multi_path = path.split(',')
1156 1156
1157 1157 if not walk:
1158 1158
1159 1159 for single_path in multi_path:
1160 1160
1161 1161 if not os.path.isdir(single_path):
1162 1162 continue
1163 1163
1164 1164 fileList = glob.glob1(single_path, "*"+ext)
1165 1165
1166 1166 if not fileList:
1167 1167 continue
1168 1168
1169 1169 path_empty = False
1170 1170
1171 1171 fileList.sort()
1172 1172
1173 1173 for thisFile in fileList:
1174 1174
1175 1175 if not os.path.isfile(os.path.join(single_path, thisFile)):
1176 1176 continue
1177 1177
1178 1178 if not isRadarFile(thisFile):
1179 1179 continue
1180 1180
1181 1181 if not isFileInDateRange(thisFile, startDate, endDate):
1182 1182 continue
1183 1183
1184 1184 thisDate = getDateFromRadarFile(thisFile)
1185 1185
1186 1186 if thisDate in dateList:
1187 1187 continue
1188 1188
1189 1189 dateList.append(thisDate)
1190 1190 pathList.append(single_path)
1191 1191
1192 1192 else:
1193 1193 for single_path in multi_path:
1194 1194
1195 1195 if not os.path.isdir(single_path):
1196 1196 continue
1197 1197
1198 1198 dirList = []
1199 1199
1200 1200 for thisPath in os.listdir(single_path):
1201 1201
1202 1202 if not os.path.isdir(os.path.join(single_path,thisPath)):
1203 1203 continue
1204 1204
1205 1205 if not isRadarFolder(thisPath):
1206 1206 continue
1207 1207
1208 1208 if not isFolderInDateRange(thisPath, startDate, endDate):
1209 1209 continue
1210 1210
1211 1211 dirList.append(thisPath)
1212 1212
1213 1213 if not dirList:
1214 1214 continue
1215 1215
1216 1216 dirList.sort()
1217 1217
1218 1218 for thisDir in dirList:
1219 1219
1220 1220 datapath = os.path.join(single_path, thisDir, expLabel)
1221 1221 fileList = glob.glob1(datapath, "*"+ext)
1222 1222
1223 1223 if not fileList:
1224 1224 continue
1225 1225
1226 1226 path_empty = False
1227 1227
1228 1228 thisDate = getDateFromRadarFolder(thisDir)
1229 1229
1230 1230 pathList.append(datapath)
1231 1231 dateList.append(thisDate)
1232 1232
1233 1233 dateList.sort()
1234 1234
1235 1235 if walk:
1236 1236 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1237 1237 else:
1238 1238 pattern_path = multi_path[0]
1239 1239
1240 1240 if path_empty:
1241 1241 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1242 1242 else:
1243 1243 if not dateList:
1244 1244 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1245 1245
1246 1246 if include_path:
1247 1247 return dateList, pathList
1248 1248
1249 1249 return dateList
1250 1250
1251 1251 def setup(self,
1252 1252 path=None,
1253 1253 startDate=None,
1254 1254 endDate=None,
1255 1255 startTime=datetime.time(0,0,0),
1256 1256 endTime=datetime.time(23,59,59),
1257 1257 set=None,
1258 1258 expLabel = "",
1259 1259 ext = None,
1260 1260 online = False,
1261 1261 delay = 60,
1262 1262 walk = True,
1263 1263 getblock = False,
1264 1264 nTxs = 1,
1265 1265 realtime=False,
1266 1266 blocksize=None,
1267 1267 blocktime=None,
1268 1268 skip=None,
1269 1269 cursor=None,
1270 1270 warnings=True,
1271 1271 verbose=True,
1272 server=None):
1272 server=None,
1273 format=None,
1274 oneDDict=None,
1275 twoDDict=None,
1276 ind2DList=None):
1273 1277 if server is not None:
1274 1278 if 'tcp://' in server:
1275 1279 address = server
1276 1280 else:
1277 1281 address = 'ipc:///tmp/%s' % server
1278 1282 self.server = address
1279 1283 self.context = zmq.Context()
1280 1284 self.receiver = self.context.socket(zmq.PULL)
1281 1285 self.receiver.connect(self.server)
1282 1286 time.sleep(0.5)
1283 1287 print '[Starting] ReceiverData from {}'.format(self.server)
1284 1288 else:
1285 1289 self.server = None
1286 1290 if path == None:
1287 1291 raise ValueError, "[Reading] The path is not valid"
1288 1292
1289 1293 if ext == None:
1290 1294 ext = self.ext
1291 1295
1292 1296 if online:
1293 1297 print "[Reading] Searching files in online mode..."
1294 1298
1295 1299 for nTries in range( self.nTries ):
1296 1300 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1297 1301
1298 1302 if fullpath:
1299 1303 break
1300 1304
1301 1305 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1302 1306 sleep( self.delay )
1303 1307
1304 1308 if not(fullpath):
1305 1309 print "[Reading] There 'isn't any valid file in %s" % path
1306 1310 return
1307 1311
1308 1312 self.year = year
1309 1313 self.doy = doy
1310 1314 self.set = set - 1
1311 1315 self.path = path
1312 1316 self.foldercounter = foldercounter
1313 1317 last_set = None
1314 1318 else:
1315 1319 print "[Reading] Searching files in offline mode ..."
1316 1320 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1317 1321 startTime=startTime, endTime=endTime,
1318 1322 set=set, expLabel=expLabel, ext=ext,
1319 1323 walk=walk, cursor=cursor,
1320 1324 skip=skip)
1321 1325
1322 1326 if not(pathList):
1323 1327 self.fileIndex = -1
1324 1328 self.pathList = []
1325 1329 self.filenameList = []
1326 1330 return
1327 1331
1328 1332 self.fileIndex = -1
1329 1333 self.pathList = pathList
1330 1334 self.filenameList = filenameList
1331 1335 file_name = os.path.basename(filenameList[-1])
1332 1336 basename, ext = os.path.splitext(file_name)
1333 1337 last_set = int(basename[-3:])
1334 1338
1335 1339 self.online = online
1336 1340 self.realtime = realtime
1337 1341 self.delay = delay
1338 1342 ext = ext.lower()
1339 1343 self.ext = ext
1340 1344 self.getByBlock = getblock
1341 1345 self.nTxs = nTxs
1342 1346 self.startTime = startTime
1343 1347 self.endTime = endTime
1344 1348
1345 1349 #Added-----------------
1346 1350 self.selBlocksize = blocksize
1347 1351 self.selBlocktime = blocktime
1348 1352
1349 1353 # Verbose-----------
1350 1354 self.verbose = verbose
1351 1355 self.warnings = warnings
1352 1356
1353 1357 if not(self.setNextFile()):
1354 1358 if (startDate!=None) and (endDate!=None):
1355 1359 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1356 1360 elif startDate != None:
1357 1361 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1358 1362 else:
1359 1363 print "[Reading] No files"
1360 1364
1361 1365 self.fileIndex = -1
1362 1366 self.pathList = []
1363 1367 self.filenameList = []
1364 1368 return
1365 1369
1366 1370 # self.getBasicHeader()
1367 1371
1368 1372 if last_set != None:
1369 1373 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1370 1374 return
1371 1375
1372 1376 def getBasicHeader(self):
1373 1377
1374 1378 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1375 1379
1376 1380 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1377 1381
1378 1382 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1379 1383
1380 1384 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1381 1385
1382 1386 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1383 1387
1384 1388 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1385 1389
1386 1390 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1387 1391
1388 1392 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1389 1393
1390 1394
1391 1395 def getFirstHeader(self):
1392 1396
1393 1397 raise NotImplementedError
1394 1398
1395 1399 def getData(self):
1396 1400
1397 1401 raise NotImplementedError
1398 1402
1399 1403 def hasNotDataInBuffer(self):
1400 1404
1401 1405 raise NotImplementedError
1402 1406
1403 1407 def readBlock(self):
1404 1408
1405 1409 raise NotImplementedError
1406 1410
1407 1411 def isEndProcess(self):
1408 1412
1409 1413 return self.flagNoMoreFiles
1410 1414
1411 1415 def printReadBlocks(self):
1412 1416
1413 1417 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1414 1418
1415 1419 def printTotalBlocks(self):
1416 1420
1417 1421 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1418 1422
1419 1423 def printNumberOfBlock(self):
1420 1424 'SPAM!'
1421 1425
1422 1426 # if self.flagIsNewBlock:
1423 1427 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1424 1428 # self.processingHeaderObj.dataBlocksPerFile,
1425 1429 # self.dataOut.datatime.ctime())
1426 1430
1427 1431 def printInfo(self):
1428 1432
1429 1433 if self.__printInfo == False:
1430 1434 return
1431 1435
1432 1436 self.basicHeaderObj.printInfo()
1433 1437 self.systemHeaderObj.printInfo()
1434 1438 self.radarControllerHeaderObj.printInfo()
1435 1439 self.processingHeaderObj.printInfo()
1436 1440
1437 1441 self.__printInfo = False
1438 1442
1439 1443 def run(self,
1440 1444 path=None,
1441 1445 startDate=None,
1442 1446 endDate=None,
1443 1447 startTime=datetime.time(0,0,0),
1444 1448 endTime=datetime.time(23,59,59),
1445 1449 set=None,
1446 1450 expLabel = "",
1447 1451 ext = None,
1448 1452 online = False,
1449 1453 delay = 60,
1450 1454 walk = True,
1451 1455 getblock = False,
1452 1456 nTxs = 1,
1453 1457 realtime=False,
1454 1458 blocksize=None,
1455 1459 blocktime=None,
1456 1460 skip=None,
1457 1461 cursor=None,
1458 1462 warnings=True,
1459 1463 server=None,
1460 verbose=True, **kwargs):
1464 verbose=True,
1465 format=None,
1466 oneDDict=None,
1467 twoDDict=None,
1468 ind2DList=None, **kwargs):
1461 1469
1462 1470 if not(self.isConfig):
1463 1471 self.setup(path=path,
1464 1472 startDate=startDate,
1465 1473 endDate=endDate,
1466 1474 startTime=startTime,
1467 1475 endTime=endTime,
1468 1476 set=set,
1469 1477 expLabel=expLabel,
1470 1478 ext=ext,
1471 1479 online=online,
1472 1480 delay=delay,
1473 1481 walk=walk,
1474 1482 getblock=getblock,
1475 1483 nTxs=nTxs,
1476 1484 realtime=realtime,
1477 1485 blocksize=blocksize,
1478 1486 blocktime=blocktime,
1479 1487 skip=skip,
1480 1488 cursor=cursor,
1481 1489 warnings=warnings,
1482 1490 server=server,
1483 verbose=verbose)
1491 verbose=verbose,
1492 format=format,
1493 oneDDict=oneDDict,
1494 twoDDict=twoDDict,
1495 ind2DList=ind2DList)
1484 1496 self.isConfig = True
1485 1497 if server is None:
1486 1498 self.getData()
1487 1499 else:
1488 1500 self.getFromServer()
1489 1501
1490 1502 class JRODataWriter(JRODataIO):
1491 1503
1492 1504 """
1493 1505 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1494 1506 de los datos siempre se realiza por bloques.
1495 1507 """
1496 1508
1497 1509 blockIndex = 0
1498 1510
1499 1511 path = None
1500 1512
1501 1513 setFile = None
1502 1514
1503 1515 profilesPerBlock = None
1504 1516
1505 1517 blocksPerFile = None
1506 1518
1507 1519 nWriteBlocks = 0
1508 1520
1509 1521 fileDate = None
1510 1522
1511 1523 def __init__(self, dataOut=None):
1512 1524 raise NotImplementedError
1513 1525
1514 1526
1515 1527 def hasAllDataInBuffer(self):
1516 1528 raise NotImplementedError
1517 1529
1518 1530
1519 1531 def setBlockDimension(self):
1520 1532 raise NotImplementedError
1521 1533
1522 1534
1523 1535 def writeBlock(self):
1524 1536 raise NotImplementedError
1525 1537
1526 1538
1527 1539 def putData(self):
1528 1540 raise NotImplementedError
1529 1541
1530 1542
1531 1543 def getProcessFlags(self):
1532 1544
1533 1545 processFlags = 0
1534 1546
1535 1547 dtype_index = get_dtype_index(self.dtype)
1536 1548 procflag_dtype = get_procflag_dtype(dtype_index)
1537 1549
1538 1550 processFlags += procflag_dtype
1539 1551
1540 1552 if self.dataOut.flagDecodeData:
1541 1553 processFlags += PROCFLAG.DECODE_DATA
1542 1554
1543 1555 if self.dataOut.flagDeflipData:
1544 1556 processFlags += PROCFLAG.DEFLIP_DATA
1545 1557
1546 1558 if self.dataOut.code is not None:
1547 1559 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1548 1560
1549 1561 if self.dataOut.nCohInt > 1:
1550 1562 processFlags += PROCFLAG.COHERENT_INTEGRATION
1551 1563
1552 1564 if self.dataOut.type == "Spectra":
1553 1565 if self.dataOut.nIncohInt > 1:
1554 1566 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1555 1567
1556 1568 if self.dataOut.data_dc is not None:
1557 1569 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1558 1570
1559 1571 if self.dataOut.flagShiftFFT:
1560 1572 processFlags += PROCFLAG.SHIFT_FFT_DATA
1561 1573
1562 1574 return processFlags
1563 1575
1564 1576 def setBasicHeader(self):
1565 1577
1566 1578 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1567 1579 self.basicHeaderObj.version = self.versionFile
1568 1580 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1569 1581
1570 1582 utc = numpy.floor(self.dataOut.utctime)
1571 1583 milisecond = (self.dataOut.utctime - utc)* 1000.0
1572 1584
1573 1585 self.basicHeaderObj.utc = utc
1574 1586 self.basicHeaderObj.miliSecond = milisecond
1575 1587 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1576 1588 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1577 1589 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1578 1590
1579 1591 def setFirstHeader(self):
1580 1592 """
1581 1593 Obtiene una copia del First Header
1582 1594
1583 1595 Affected:
1584 1596
1585 1597 self.basicHeaderObj
1586 1598 self.systemHeaderObj
1587 1599 self.radarControllerHeaderObj
1588 1600 self.processingHeaderObj self.
1589 1601
1590 1602 Return:
1591 1603 None
1592 1604 """
1593 1605
1594 1606 raise NotImplementedError
1595 1607
1596 1608 def __writeFirstHeader(self):
1597 1609 """
1598 1610 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1599 1611
1600 1612 Affected:
1601 1613 __dataType
1602 1614
1603 1615 Return:
1604 1616 None
1605 1617 """
1606 1618
1607 1619 # CALCULAR PARAMETROS
1608 1620
1609 1621 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1610 1622 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1611 1623
1612 1624 self.basicHeaderObj.write(self.fp)
1613 1625 self.systemHeaderObj.write(self.fp)
1614 1626 self.radarControllerHeaderObj.write(self.fp)
1615 1627 self.processingHeaderObj.write(self.fp)
1616 1628
1617 1629 def __setNewBlock(self):
1618 1630 """
1619 1631 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1620 1632
1621 1633 Return:
1622 1634 0 : si no pudo escribir nada
1623 1635 1 : Si escribio el Basic el First Header
1624 1636 """
1625 1637 if self.fp == None:
1626 1638 self.setNextFile()
1627 1639
1628 1640 if self.flagIsNewFile:
1629 1641 return 1
1630 1642
1631 1643 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1632 1644 self.basicHeaderObj.write(self.fp)
1633 1645 return 1
1634 1646
1635 1647 if not( self.setNextFile() ):
1636 1648 return 0
1637 1649
1638 1650 return 1
1639 1651
1640 1652
1641 1653 def writeNextBlock(self):
1642 1654 """
1643 1655 Selecciona el bloque siguiente de datos y los escribe en un file
1644 1656
1645 1657 Return:
1646 1658 0 : Si no hizo pudo escribir el bloque de datos
1647 1659 1 : Si no pudo escribir el bloque de datos
1648 1660 """
1649 1661 if not( self.__setNewBlock() ):
1650 1662 return 0
1651 1663
1652 1664 self.writeBlock()
1653 1665
1654 1666 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1655 1667 self.processingHeaderObj.dataBlocksPerFile)
1656 1668
1657 1669 return 1
1658 1670
1659 1671 def setNextFile(self):
1660 1672 """
1661 1673 Determina el siguiente file que sera escrito
1662 1674
1663 1675 Affected:
1664 1676 self.filename
1665 1677 self.subfolder
1666 1678 self.fp
1667 1679 self.setFile
1668 1680 self.flagIsNewFile
1669 1681
1670 1682 Return:
1671 1683 0 : Si el archivo no puede ser escrito
1672 1684 1 : Si el archivo esta listo para ser escrito
1673 1685 """
1674 1686 ext = self.ext
1675 1687 path = self.path
1676 1688
1677 1689 if self.fp != None:
1678 1690 self.fp.close()
1679 1691
1680 1692 timeTuple = time.localtime( self.dataOut.utctime)
1681 1693 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1682 1694
1683 1695 fullpath = os.path.join( path, subfolder )
1684 1696 setFile = self.setFile
1685 1697
1686 1698 if not( os.path.exists(fullpath) ):
1687 1699 os.mkdir(fullpath)
1688 1700 setFile = -1 #inicializo mi contador de seteo
1689 1701 else:
1690 1702 filesList = os.listdir( fullpath )
1691 1703 if len( filesList ) > 0:
1692 1704 filesList = sorted( filesList, key=str.lower )
1693 1705 filen = filesList[-1]
1694 1706 # el filename debera tener el siguiente formato
1695 1707 # 0 1234 567 89A BCDE (hex)
1696 1708 # x YYYY DDD SSS .ext
1697 1709 if isNumber( filen[8:11] ):
1698 1710 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1699 1711 else:
1700 1712 setFile = -1
1701 1713 else:
1702 1714 setFile = -1 #inicializo mi contador de seteo
1703 1715
1704 1716 setFile += 1
1705 1717
1706 1718 #If this is a new day it resets some values
1707 1719 if self.dataOut.datatime.date() > self.fileDate:
1708 1720 setFile = 0
1709 1721 self.nTotalBlocks = 0
1710 1722
1711 1723 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1712 1724
1713 1725 filename = os.path.join( path, subfolder, filen )
1714 1726
1715 1727 fp = open( filename,'wb' )
1716 1728
1717 1729 self.blockIndex = 0
1718 1730
1719 1731 #guardando atributos
1720 1732 self.filename = filename
1721 1733 self.subfolder = subfolder
1722 1734 self.fp = fp
1723 1735 self.setFile = setFile
1724 1736 self.flagIsNewFile = 1
1725 1737 self.fileDate = self.dataOut.datatime.date()
1726 1738
1727 1739 self.setFirstHeader()
1728 1740
1729 1741 print '[Writing] Opening file: %s'%self.filename
1730 1742
1731 1743 self.__writeFirstHeader()
1732 1744
1733 1745 return 1
1734 1746
1735 1747 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1736 1748 """
1737 1749 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1738 1750
1739 1751 Inputs:
1740 1752 path : directory where data will be saved
1741 1753 profilesPerBlock : number of profiles per block
1742 1754 set : initial file set
1743 1755 datatype : An integer number that defines data type:
1744 1756 0 : int8 (1 byte)
1745 1757 1 : int16 (2 bytes)
1746 1758 2 : int32 (4 bytes)
1747 1759 3 : int64 (8 bytes)
1748 1760 4 : float32 (4 bytes)
1749 1761 5 : double64 (8 bytes)
1750 1762
1751 1763 Return:
1752 1764 0 : Si no realizo un buen seteo
1753 1765 1 : Si realizo un buen seteo
1754 1766 """
1755 1767
1756 1768 if ext == None:
1757 1769 ext = self.ext
1758 1770
1759 1771 self.ext = ext.lower()
1760 1772
1761 1773 self.path = path
1762 1774
1763 1775 if set is None:
1764 1776 self.setFile = -1
1765 1777 else:
1766 1778 self.setFile = set - 1
1767 1779
1768 1780 self.blocksPerFile = blocksPerFile
1769 1781
1770 1782 self.profilesPerBlock = profilesPerBlock
1771 1783
1772 1784 self.dataOut = dataOut
1773 1785 self.fileDate = self.dataOut.datatime.date()
1774 1786 #By default
1775 1787 self.dtype = self.dataOut.dtype
1776 1788
1777 1789 if datatype is not None:
1778 1790 self.dtype = get_numpy_dtype(datatype)
1779 1791
1780 1792 if not(self.setNextFile()):
1781 1793 print "[Writing] There isn't a next file"
1782 1794 return 0
1783 1795
1784 1796 self.setBlockDimension()
1785 1797
1786 1798 return 1
1787 1799
1788 1800 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1789 1801
1790 1802 if not(self.isConfig):
1791 1803
1792 1804 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1793 1805 self.isConfig = True
1794 1806
1795 1807 self.putData()
Show file before | Show file after | Hide comments
@@ -1,243 +1,580
1 1 '''
2 2 Created on Aug 1, 2017
3 3
4 4 @author: Juan C. Espinoza
5 5 '''
6 6
7 7 import os
8 8 import sys
9 9 import time
10 10 import json
11 import glob
11 12 import datetime
12 13
13 14 import numpy
15 import h5py
14 16
15 17 try:
16 18 import madrigal
17 19 import madrigal.cedar
18 20 except:
19 21 print 'You should install "madrigal library" module if you want to read/write Madrigal data'
20 22
21 from schainpy.model.proc.jroproc_base import Operation
23 from schainpy.model.io.jroIO_base import JRODataReader
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
22 25 from schainpy.model.data.jrodata import Parameters
26 from schainpy.utils import log
27
23 28
24 MISSING = -32767
25 29 DEF_CATALOG = {
26 30 'principleInvestigator': 'Marco Milla',
27 31 'expPurpose': None,
28 32 'expMode': None,
29 33 'cycleTime': None,
30 34 'correlativeExp': None,
31 35 'sciRemarks': None,
32 36 'instRemarks': None
33 37 }
34 38 DEF_HEADER = {
35 39 'kindatDesc': None,
36 40 'analyst': 'Jicamarca User',
37 41 'comments': None,
38 42 'history': None
39 43 }
40 44 MNEMONICS = {
41 45 10: 'jro',
42 46 11: 'jbr',
43 47 840: 'jul',
44 48 13: 'jas',
45 49 1000: 'pbr',
46 50 1001: 'hbr',
47 51 1002: 'obr',
48 52 }
49 53
54 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
55
50 56 def load_json(obj):
51 57 '''
52 58 Parse json as string instead of unicode
53 59 '''
54 60
55 61 if isinstance(obj, str):
56 obj = json.loads(obj)
62 iterable = json.loads(obj)
63
64 if isinstance(iterable, dict):
65 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
66 for k, v in iterable.items()}
67 elif isinstance(iterable, (list, tuple)):
68 return [str(v) if isinstance(v, unicode) else v for v in iterable]
69
70 return iterable
71
72
73 class MADReader(JRODataReader, ProcessingUnit):
74
75 def __init__(self, **kwargs):
76
77 ProcessingUnit.__init__(self, **kwargs)
78
79 self.dataOut = Parameters()
80 self.counter_records = 0
81 self.nrecords = None
82 self.flagNoMoreFiles = 0
83 self.isConfig = False
84 self.filename = None
85 self.intervals = set()
86
87 def setup(self,
88 path=None,
89 startDate=None,
90 endDate=None,
91 format=None,
92 startTime=datetime.time(0, 0, 0),
93 endTime=datetime.time(23, 59, 59),
94 **kwargs):
95
96 self.started = True
97 self.path = path
98 self.startDate = startDate
99 self.endDate = endDate
100 self.startTime = startTime
101 self.endTime = endTime
102 self.datatime = datetime.datetime(1900,1,1)
103 self.oneDDict = load_json(kwargs.get('oneDDict',
104 "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
105 self.twoDDict = load_json(kwargs.get('twoDDict',
106 "{\"GDALT\": \"heightList\"}"))
107 self.ind2DList = load_json(kwargs.get('ind2DList',
108 "[\"GDALT\"]"))
109 if self.path is None:
110 raise ValueError, 'The path is not valid'
111
112 if format is None:
113 raise ValueError, 'The format is not valid choose simple or hdf5'
114 elif format.lower() in ('simple', 'txt'):
115 self.ext = '.txt'
116 elif format.lower() in ('cedar',):
117 self.ext = '.001'
118 else:
119 self.ext = '.hdf5'
120
121 self.search_files(self.path)
122 self.fileId = 0
123
124 if not self.fileList:
125 raise Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path)
126
127 self.setNextFile()
128
129 def search_files(self, path):
130 '''
131 Searching for madrigal files in path
132 Creating a list of files to procces included in [startDate,endDate]
133
134 Input:
135 path - Path to find files
136 '''
137
138 print 'Searching files {} in {} '.format(self.ext, path)
139 foldercounter = 0
140 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
141 fileList0.sort()
142
143 self.fileList = []
144 self.dateFileList = []
57 145
58 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
59 for k, v in obj.items()}
146 startDate = self.startDate - datetime.timedelta(1)
147 endDate = self.endDate + datetime.timedelta(1)
148
149 for thisFile in fileList0:
150 year = thisFile[3:7]
151 if not year.isdigit():
152 continue
153
154 month = thisFile[7:9]
155 if not month.isdigit():
156 continue
157
158 day = thisFile[9:11]
159 if not day.isdigit():
160 continue
161
162 year, month, day = int(year), int(month), int(day)
163 dateFile = datetime.date(year, month, day)
164
165 if (startDate > dateFile) or (endDate < dateFile):
166 continue
167
168 self.fileList.append(thisFile)
169 self.dateFileList.append(dateFile)
170
171 return
172
173 def parseHeader(self):
174 '''
175 '''
176
177 self.output = {}
178 self.version = '2'
179 s_parameters = None
180 if self.ext == '.txt':
181 self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s]
182 elif self.ext == '.hdf5':
183 metadata = self.fp['Metadata']
184 data = self.fp['Data']['Array Layout']
185 if 'Independent Spatial Parameters' in metadata:
186 s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']]
187 self.version = '3'
188 one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']]
189 one_d = [1 for s in one]
190 two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']]
191 two_d = [2 for s in two]
192 self.parameters = one + two
193 self.parameters_d = one_d + two_d
194
195 log.success('Parameters found: {}'.format(','.join(self.parameters)),
196 'MADReader')
197 if s_parameters:
198 log.success('Spatial parameters: {}'.format(','.join(s_parameters)),
199 'MADReader')
200
201 for param in self.oneDDict.keys():
202 if param.lower() not in self.parameters:
203 print('\x1b[33m[Warning]\x1b[0m Parameter \x1b[1;32m{}\x1b[0m not found will be ignored'.format(
204 param
205 ))
206 self.oneDDict.pop(param, None)
207
208 for param, value in self.twoDDict.items():
209 if param.lower() not in self.parameters:
210 print('\x1b[33m[Warning]\x1b[0m Parameter \x1b[1;32m{}\x1b[0m not found will be ignored'.format(
211 param
212 ))
213 self.twoDDict.pop(param, None)
214 continue
215 if isinstance(value, list):
216 if value[0] not in self.output:
217 self.output[value[0]] = []
218 self.output[value[0]].append(None)
219
220 def parseData(self):
221 '''
222 '''
223
224 if self.ext == '.txt':
225 self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
226 self.nrecords = self.data.shape[0]
227 self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())])
228 elif self.ext == '.hdf5':
229 self.data = self.fp['Data']['Array Layout']
230 self.nrecords = len(self.data['timestamps'].value)
231 self.ranges = self.data['range'].value
232
233 def setNextFile(self):
234 '''
235 '''
236
237 file_id = self.fileId
238
239 if file_id == len(self.fileList):
240 print '\nNo more files in the folder'
241 print 'Total number of file(s) read : {}'.format(self.fileId)
242 self.flagNoMoreFiles = 1
243 return 0
244
245 print('\x1b[32m[Info]\x1b[0m Opening: {}'.format(
246 self.fileList[file_id]
247 ))
248 filename = os.path.join(self.path, self.fileList[file_id])
249
250 if self.filename is not None:
251 self.fp.close()
252
253 self.filename = filename
254 self.filedate = self.dateFileList[file_id]
255
256 if self.ext=='.hdf5':
257 self.fp = h5py.File(self.filename, 'r')
258 else:
259 self.fp = open(self.filename, 'rb')
260
261 self.parseHeader()
262 self.parseData()
263 self.sizeOfFile = os.path.getsize(self.filename)
264 self.counter_records = 0
265 self.flagIsNewFile = 0
266 self.fileId += 1
267
268 return 1
269
270 def readNextBlock(self):
271
272 while True:
273
274 if self.flagIsNewFile:
275 if not self.setNextFile():
276 return 0
277
278 self.readBlock()
279
280 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
281 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
282 print "\x1b[32m[Reading]\x1b[0m Record No. %d/%d -> %s \x1b[33m[Skipping]\x1b[0m" %(
283 self.counter_records,
284 self.nrecords,
285 self.datatime.ctime())
286 continue
287 break
288
289 print "\x1b[32m[Reading]\x1b[0m Record No. %d/%d -> %s" %(
290 self.counter_records,
291 self.nrecords,
292 self.datatime.ctime())
293
294 return 1
295
296 def readBlock(self):
297 '''
298 '''
299 dum = []
300 if self.ext == '.txt':
301 dt = self.data[self.counter_records][:6].astype(int)
302 self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
303 while True:
304 dt = self.data[self.counter_records][:6].astype(int)
305 datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
306 if datatime == self.datatime:
307 dum.append(self.data[self.counter_records])
308 self.counter_records += 1
309 if self.counter_records == self.nrecords:
310 self.flagIsNewFile = True
311 break
312 continue
313 self.intervals.add((datatime-self.datatime).seconds)
314 break
315 elif self.ext == '.hdf5':
316 datatime = datetime.datetime.utcfromtimestamp(
317 self.data['timestamps'][self.counter_records])
318 nHeights = len(self.ranges)
319 for n, param in enumerate(self.parameters):
320 if self.parameters_d[n] == 1:
321 dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records])
322 else:
323 if self.version == '2':
324 dum.append(self.data['2D Parameters'][param][self.counter_records])
325 else:
326 tmp = self.data['2D Parameters'][param].value.T
327 dum.append(tmp[self.counter_records])
328 self.intervals.add((datatime-self.datatime).seconds)
329 self.datatime = datatime
330 self.counter_records += 1
331 if self.counter_records == self.nrecords:
332 self.flagIsNewFile = True
333
334 self.buffer = numpy.array(dum)
335 return
336
337 def set_output(self):
338 '''
339 Storing data from buffer to dataOut object
340 '''
341
342 parameters = [None for __ in self.parameters]
343
344 for param, attr in self.oneDDict.items():
345 x = self.parameters.index(param.lower())
346 setattr(self.dataOut, attr, self.buffer[0][x])
347
348 for param, value in self.twoDDict.items():
349 x = self.parameters.index(param.lower())
350 if self.ext == '.txt':
351 y = self.parameters.index(self.ind2DList[0].lower())
352 ranges = self.buffer[:,y]
353 if self.ranges.size == ranges.size:
354 continue
355 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
356 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
357 dummy[index] = self.buffer[:,x]
358 else:
359
360 dummy = self.buffer[x]
361
362 if isinstance(value, str):
363 if value not in self.ind2DList:
364 setattr(self.dataOut, value, dummy.reshape(1,-1))
365 elif isinstance(value, list):
366 self.output[value[0]][value[1]] = dummy
367 parameters[value[1]] = param
368
369 for key, value in self.output.items():
370 setattr(self.dataOut, key, numpy.array(value))
371
372 self.dataOut.parameters = [s for s in parameters if s]
373 self.dataOut.heightList = self.ranges
374 self.dataOut.utctime = (self.datatime - UT1970).total_seconds()
375 self.dataOut.utctimeInit = self.dataOut.utctime
376 self.dataOut.paramInterval = min(self.intervals)
377 self.dataOut.useLocalTime = False
378 self.dataOut.flagNoData = False
379 self.dataOut.started = self.started
380
381 def getData(self):
382 '''
383 Storing data from databuffer to dataOut object
384 '''
385 if self.flagNoMoreFiles:
386 self.dataOut.flagNoData = True
387 print 'No file left to process'
388 return 0
389
390 if not self.readNextBlock():
391 self.dataOut.flagNoData = True
392 return 0
393
394 self.set_output()
395
396 return 1
60 397
61 398
62 399 class MAD2Writer(Operation):
400
401 missing = -32767
402 ext = '.dat'
63 403
64 404 def __init__(self, **kwargs):
65 405
66 406 Operation.__init__(self, **kwargs)
67 407 self.dataOut = Parameters()
68 408 self.path = None
69 409 self.dataOut = None
70 self.ext = '.dat'
71
72 return
73 410
74 def run(self, dataOut, path, oneDList, twoDParam='', twoDList='{}', metadata='{}', **kwargs):
411 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}', metadata='{}', **kwargs):
75 412 '''
76 413 Inputs:
77 414 path - path where files will be created
78 oneDList - json of one-dimensional parameters in record where keys
415 oneDDict - json of one-dimensional parameters in record where keys
79 416 are Madrigal codes (integers or mnemonics) and values the corresponding
80 417 dataOut attribute e.g: {
81 418 'gdlatr': 'lat',
82 419 'gdlonr': 'lon',
83 420 'gdlat2':'lat',
84 421 'glon2':'lon'}
85 twoDParam - independent parameter to get the number of rows e.g:
86 heighList
87 twoDList - json of two-dimensional parameters in record where keys
422 ind2DList - list of independent spatial two-dimensional parameters e.g:
423 ['heighList']
424 twoDDict - json of two-dimensional parameters in record where keys
88 425 are Madrigal codes (integers or mnemonics) and values the corresponding
89 426 dataOut attribute if multidimensional array specify as tupple
90 427 ('attr', pos) e.g: {
91 428 'gdalt': 'heightList',
92 429 'vn1p2': ('data_output', 0),
93 430 'vn2p2': ('data_output', 1),
94 431 'vn3': ('data_output', 2),
95 432 'snl': ('data_SNR', 'db')
96 433 }
97 434 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
98 435 '''
99 436 if not self.isConfig:
100 self.setup(dataOut, path, oneDList, twoDParam, twoDList, metadata, **kwargs)
437 self.setup(dataOut, path, oneDDict, ind2DList, twoDDict, metadata, **kwargs)
101 438 self.isConfig = True
102 439
103 440 self.putData()
104 441 return
105 442
106 def setup(self, dataOut, path, oneDList, twoDParam, twoDList, metadata, **kwargs):
443 def setup(self, dataOut, path, oneDDict, ind2DList, twoDDict, metadata, **kwargs):
107 444 '''
108 445 Configure Operation
109 446 '''
110 447
111 448 self.dataOut = dataOut
112 449 self.nmodes = self.dataOut.nmodes
113 450 self.path = path
114 451 self.blocks = kwargs.get('blocks', None)
115 452 self.counter = 0
116 self.oneDList = load_json(oneDList)
117 self.twoDList = load_json(twoDList)
118 self.twoDParam = twoDParam
453 self.oneDDict = load_json(oneDDict)
454 self.twoDDict = load_json(twoDDict)
455 self.ind2DList = load_json(ind2DList)
119 456 meta = load_json(metadata)
120 457 self.kinst = meta.get('kinst')
121 458 self.kindat = meta.get('kindat')
122 459 self.catalog = meta.get('catalog', DEF_CATALOG)
123 460 self.header = meta.get('header', DEF_HEADER)
124 461
125 462 return
126 463
127 464 def setFile(self):
128 465 '''
129 466 Create new cedar file object
130 467 '''
131 468
132 469 self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
133 470 date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
134 471
135 472 filename = '%s%s_%s%s' % (self.mnemonic,
136 473 date.strftime('%Y%m%d_%H%M%S'),
137 474 self.dataOut.mode,
138 475 self.ext)
139 476
140 477 self.fullname = os.path.join(self.path, filename)
141 478
142 479 if os.path.isfile(self.fullname) :
143 480 print "Destination path '%s' already exists. Previous file deleted. " %self.fullname
144 481 os.remove(self.fullname)
145 482
146 483 try:
147 484 print '[Writing] creating file : %s' % (self.fullname)
148 485 self.cedarObj = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
149 486 except ValueError, e:
150 487 print '[Error]: Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile" '
151 488 return
152 489
153 490 return 1
154 491
155 492 def writeBlock(self):
156 493 '''
157 Add data records to cedar file taking data from oneDList and twoDList
494 Add data records to cedar file taking data from oneDDict and twoDDict
158 495 attributes.
159 496 Allowed parameters in: parcodes.tab
160 497 '''
161 498
162 499 startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
163 500 endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
164 nrows = len(getattr(self.dataOut, self.twoDParam))
501 nrows = len(getattr(self.dataOut, self.ind2DList))
165 502
166 503 rec = madrigal.cedar.MadrigalDataRecord(
167 504 self.kinst,
168 505 self.kindat,
169 506 startTime.year,
170 507 startTime.month,
171 508 startTime.day,
172 509 startTime.hour,
173 510 startTime.minute,
174 511 startTime.second,
175 512 startTime.microsecond/10000,
176 513 endTime.year,
177 514 endTime.month,
178 515 endTime.day,
179 516 endTime.hour,
180 517 endTime.minute,
181 518 endTime.second,
182 519 endTime.microsecond/10000,
183 self.oneDList.keys(),
184 self.twoDList.keys(),
520 self.oneDDict.keys(),
521 self.twoDDict.keys(),
185 522 nrows
186 523 )
187 524
188 525 # Setting 1d values
189 for key in self.oneDList:
190 rec.set1D(key, getattr(self.dataOut, self.oneDList[key]))
526 for key in self.oneDDict:
527 rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
191 528
192 529 # Setting 2d values
193 530 invalid = numpy.isnan(self.dataOut.data_output)
194 self.dataOut.data_output[invalid] = MISSING
531 self.dataOut.data_output[invalid] = self.missing
195 532 out = {}
196 for key, value in self.twoDList.items():
533 for key, value in self.twoDDict.items():
197 534 if isinstance(value, str):
198 535 out[key] = getattr(self.dataOut, value)
199 536 elif isinstance(value, tuple):
200 537 attr, x = value
201 538 if isinstance(x, (int, float)):
202 539 out[key] = getattr(self.dataOut, attr)[int(x)]
203 540 elif x.lower()=='db':
204 541 tmp = getattr(self.dataOut, attr)
205 542 SNRavg = numpy.average(tmp, axis=0)
206 543 out[key] = 10*numpy.log10(SNRavg)
207 544
208 545 for n in range(nrows):
209 546 for key in out:
210 547 rec.set2D(key, n, out[key][n])
211 548
212 549 self.cedarObj.append(rec)
213 550 self.cedarObj.dump()
214 551 print '[Writing] Record No. {} (mode {}).'.format(
215 552 self.counter,
216 553 self.dataOut.mode
217 554 )
218 555
219 556 def setHeader(self):
220 557 '''
221 558 Create an add catalog and header to cedar file
222 559 '''
223 560
224 561 header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
225 562 header.createCatalog(**self.catalog)
226 563 header.createHeader(**self.header)
227 564 header.write()
228 565
229 566 def putData(self):
230 567
231 568 if self.dataOut.flagNoData:
232 569 return 0
233 570
234 571 if self.counter == 0:
235 572 self.setFile()
236 573
237 574 if self.counter <= self.dataOut.nrecords:
238 575 self.writeBlock()
239 576 self.counter += 1
240 577
241 578 if self.counter == self.dataOut.nrecords or self.counter == self.blocks:
242 579 self.setHeader()
243 580 self.counter = 0
Show file before | Show file after | Hide comments
@@ -1,707 +1,698
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import numpy
7 7
8 8 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
9 9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 10 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
11 11 from schainpy.model.data.jrodata import Spectra
12 12
13 13 class SpectraReader(JRODataReader, ProcessingUnit):
14 14
15 15 """
16 16 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
17 17 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
18 18 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
19 19
20 20 paresCanalesIguales * alturas * perfiles (Self Spectra)
21 21 paresCanalesDiferentes * alturas * perfiles (Cross Spectra)
22 22 canales * alturas (DC Channels)
23 23
24 24
25 25 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
26 26 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
27 27 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
28 28 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
29 29
30 30 Example:
31 31 dpath = "/home/myuser/data"
32 32
33 33 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
34 34
35 35 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
36 36
37 37 readerObj = SpectraReader()
38 38
39 39 readerObj.setup(dpath, startTime, endTime)
40 40
41 41 while(True):
42 42
43 43 readerObj.getData()
44 44
45 45 print readerObj.data_spc
46 46
47 47 print readerObj.data_cspc
48 48
49 49 print readerObj.data_dc
50 50
51 51 if readerObj.flagNoMoreFiles:
52 52 break
53 53
54 54 """
55 55
56 56 pts2read_SelfSpectra = 0
57 57
58 58 pts2read_CrossSpectra = 0
59 59
60 60 pts2read_DCchannels = 0
61 61
62 62 ext = ".pdata"
63 63
64 64 optchar = "P"
65 65
66 66 dataOut = None
67 67
68 68 nRdChannels = None
69 69
70 70 nRdPairs = None
71 71
72 72 rdPairList = []
73 73
74 74 def __init__(self, **kwargs):
75 75 """
76 76 Inicializador de la clase SpectraReader para la lectura de datos de espectros.
77 77
78 78 Inputs:
79 79
80 80 dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para
81 81 almacenar un perfil de datos cada vez que se haga un requerimiento
82 82 (getData). El perfil sera obtenido a partir del buffer de datos,
83 83 si el buffer esta vacio se hara un nuevo proceso de lectura de un
84 84 bloque de datos.
85 85 Si este parametro no es pasado se creara uno internamente.
86 86
87 87
88 88 Affected:
89 89
90 90 self.dataOut
91 91
92 92 Return : None
93 93 """
94 94
95 95
96 96 #Eliminar de la base la herencia
97 97 ProcessingUnit.__init__(self, **kwargs)
98 98
99 99 # self.isConfig = False
100 100
101 101 self.pts2read_SelfSpectra = 0
102 102
103 103 self.pts2read_CrossSpectra = 0
104 104
105 105 self.pts2read_DCchannels = 0
106 106
107 107 self.datablock = None
108 108
109 109 self.utc = None
110 110
111 111 self.ext = ".pdata"
112 112
113 113 self.optchar = "P"
114 114
115 115 self.basicHeaderObj = BasicHeader(LOCALTIME)
116 116
117 117 self.systemHeaderObj = SystemHeader()
118 118
119 119 self.radarControllerHeaderObj = RadarControllerHeader()
120 120
121 121 self.processingHeaderObj = ProcessingHeader()
122 122
123 123 self.online = 0
124 124
125 125 self.fp = None
126 126
127 127 self.idFile = None
128 128
129 129 self.dtype = None
130 130
131 131 self.fileSizeByHeader = None
132 132
133 133 self.filenameList = []
134 134
135 135 self.filename = None
136 136
137 137 self.fileSize = None
138 138
139 139 self.firstHeaderSize = 0
140 140
141 141 self.basicHeaderSize = 24
142 142
143 143 self.pathList = []
144 144
145 145 self.lastUTTime = 0
146 146
147 147 self.maxTimeStep = 30
148 148
149 149 self.flagNoMoreFiles = 0
150 150
151 151 self.set = 0
152 152
153 153 self.path = None
154 154
155 155 self.delay = 60 #seconds
156 156
157 157 self.nTries = 3 #quantity tries
158 158
159 159 self.nFiles = 3 #number of files for searching
160 160
161 161 self.nReadBlocks = 0
162 162
163 163 self.flagIsNewFile = 1
164 164
165 165 self.__isFirstTimeOnline = 1
166 166
167 167 # self.ippSeconds = 0
168 168
169 169 self.flagDiscontinuousBlock = 0
170 170
171 171 self.flagIsNewBlock = 0
172 172
173 173 self.nTotalBlocks = 0
174 174
175 175 self.blocksize = 0
176 176
177 177 self.dataOut = self.createObjByDefault()
178 178
179 179 self.profileIndex = 1 #Always
180 180
181 181
182 182 def createObjByDefault(self):
183 183
184 184 dataObj = Spectra()
185 185
186 186 return dataObj
187 187
188 188 def __hasNotDataInBuffer(self):
189 189 return 1
190 190
191 191
192 192 def getBlockDimension(self):
193 193 """
194 194 Obtiene la cantidad de puntos a leer por cada bloque de datos
195 195
196 196 Affected:
197 197 self.nRdChannels
198 198 self.nRdPairs
199 199 self.pts2read_SelfSpectra
200 200 self.pts2read_CrossSpectra
201 201 self.pts2read_DCchannels
202 202 self.blocksize
203 203 self.dataOut.nChannels
204 204 self.dataOut.nPairs
205 205
206 206 Return:
207 207 None
208 208 """
209 209 self.nRdChannels = 0
210 210 self.nRdPairs = 0
211 211 self.rdPairList = []
212 212
213 213 for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
214 214 if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
215 215 self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
216 216
217 217 else:
218 218 self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
219 219 self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
220 220
221 221 pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
222 222
223 223 self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
224 224 self.blocksize = self.pts2read_SelfSpectra
225 225
226 226 if self.processingHeaderObj.flag_cspc:
227 227 self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
228 228 self.blocksize += self.pts2read_CrossSpectra
229 229
230 230 if self.processingHeaderObj.flag_dc:
231 231 self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
232 232 self.blocksize += self.pts2read_DCchannels
233 233
234 234 # self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
235 235
236 236
237 237 def readBlock(self):
238 238 """
239 239 Lee el bloque de datos desde la posicion actual del puntero del archivo
240 240 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
241 241 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
242 242 es seteado a 0
243 243
244 244 Return: None
245 245
246 246 Variables afectadas:
247 247
248 248
249 249 self.flagIsNewFile
250 250 self.flagIsNewBlock
251 251 self.nTotalBlocks
252 252 self.data_spc
253 253 self.data_cspc
254 254 self.data_dc
255 255
256 256 Exceptions:
257 257 Si un bloque leido no es un bloque valido
258 258 """
259 print ' ======================================================== '
260 print ' '
261 print ' '
262 print self.processingHeaderObj.totalSpectra, 'TotalSpectra', type(self.processingHeaderObj.totalSpectra)
263 print self.processingHeaderObj.spectraComb, 'SpectraComb', type(self.processingHeaderObj.spectraComb)
264 print ' '
265 print ' '
266 print ' ======================================================== '
267
268 259
269 260 blockOk_flag = False
270 261 fpointer = self.fp.tell()
271 262
272 263 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
273 264 spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
274 265
275 266 if self.processingHeaderObj.flag_cspc:
276 267 cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
277 268 cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
278 269
279 270 if self.processingHeaderObj.flag_dc:
280 271 dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
281 272 dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
282 273
283 274
284 275 if not(self.processingHeaderObj.shif_fft):
285 276 #desplaza a la derecha en el eje 2 determinadas posiciones
286 277 shift = int(self.processingHeaderObj.profilesPerBlock/2)
287 278 spc = numpy.roll( spc, shift , axis=2 )
288 279
289 280 if self.processingHeaderObj.flag_cspc:
290 281 #desplaza a la derecha en el eje 2 determinadas posiciones
291 282 cspc = numpy.roll( cspc, shift, axis=2 )
292 283
293 284 #Dimensions : nChannels, nProfiles, nSamples
294 285 spc = numpy.transpose( spc, (0,2,1) )
295 286 self.data_spc = spc
296 287
297 288 if self.processingHeaderObj.flag_cspc:
298 289
299 290 cspc = numpy.transpose( cspc, (0,2,1) )
300 291 self.data_cspc = cspc['real'] + cspc['imag']*1j
301 292 else:
302 293 self.data_cspc = None
303 294
304 295
305 296 if self.processingHeaderObj.flag_dc:
306 297 self.data_dc = dc['real'] + dc['imag']*1j
307 298 else:
308 299 self.data_dc = None
309 300
310 301 self.flagIsNewFile = 0
311 302 self.flagIsNewBlock = 1
312 303
313 304 self.nTotalBlocks += 1
314 305 self.nReadBlocks += 1
315 306
316 307 return 1
317 308
318 309 def getFirstHeader(self):
319 310
320 311 self.getBasicHeader()
321 312
322 313 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
323 314
324 315 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
325 316
326 317 # self.dataOut.ippSeconds = self.ippSeconds
327 318
328 319 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.processingHeaderObj.profilesPerBlock
329 320
330 321 self.dataOut.dtype = self.dtype
331 322
332 323 # self.dataOut.nPairs = self.nPairs
333 324
334 325 self.dataOut.pairsList = self.rdPairList
335 326
336 327 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
337 328
338 329 self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
339 330
340 331 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
341 332
342 333 self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
343 334
344 335 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
345 336
346 337 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
347 338
348 339 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
349 340
350 341 self.dataOut.flagShiftFFT = True #Data is always shifted
351 342
352 343 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
353 344
354 345 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data esta sin flip
355 346
356 347 def getData(self):
357 348 """
358 349 First method to execute before "RUN" is called.
359 350
360 351 Copia el buffer de lectura a la clase "Spectra",
361 352 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
362 353 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
363 354
364 355 Return:
365 356 0 : Si no hay mas archivos disponibles
366 357 1 : Si hizo una buena copia del buffer
367 358
368 359 Affected:
369 360 self.dataOut
370 361
371 362 self.flagDiscontinuousBlock
372 363 self.flagIsNewBlock
373 364 """
374 365
375 366 if self.flagNoMoreFiles:
376 367 self.dataOut.flagNoData = True
377 368 print 'Process finished'
378 369 return 0
379 370
380 371 self.flagDiscontinuousBlock = 0
381 372 self.flagIsNewBlock = 0
382 373
383 374 if self.__hasNotDataInBuffer():
384 375
385 376 if not( self.readNextBlock() ):
386 377 self.dataOut.flagNoData = True
387 378 return 0
388 379
389 380
390 381 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
391 382
392 383 if self.data_spc is None:
393 384 self.dataOut.flagNoData = True
394 385 return 0
395 386
396 387 self.getBasicHeader()
397 388
398 389 self.getFirstHeader()
399 390
400 391 self.dataOut.data_spc = self.data_spc
401 392
402 393 self.dataOut.data_cspc = self.data_cspc
403 394
404 395 self.dataOut.data_dc = self.data_dc
405 396
406 397 self.dataOut.flagNoData = False
407 398
408 399 self.dataOut.realtime = self.online
409 400
410 401 return self.dataOut.data_spc
411 402
412 403 class SpectraWriter(JRODataWriter, Operation):
413 404
414 405 """
415 406 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
416 407 de los datos siempre se realiza por bloques.
417 408 """
418 409
419 410 ext = ".pdata"
420 411
421 412 optchar = "P"
422 413
423 414 shape_spc_Buffer = None
424 415
425 416 shape_cspc_Buffer = None
426 417
427 418 shape_dc_Buffer = None
428 419
429 420 data_spc = None
430 421
431 422 data_cspc = None
432 423
433 424 data_dc = None
434 425
435 426 # dataOut = None
436 427
437 428 def __init__(self):
438 429 """
439 430 Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
440 431
441 432 Affected:
442 433
443 434 self.dataOut
444 435 self.basicHeaderObj
445 436 self.systemHeaderObj
446 437 self.radarControllerHeaderObj
447 438 self.processingHeaderObj
448 439
449 440 Return: None
450 441 """
451 442
452 443 Operation.__init__(self)
453 444
454 445 self.isConfig = False
455 446
456 447 self.nTotalBlocks = 0
457 448
458 449 self.data_spc = None
459 450
460 451 self.data_cspc = None
461 452
462 453
463 454 self.data_dc = None
464 455
465 456 self.fp = None
466 457
467 458 self.flagIsNewFile = 1
468 459
469 460 self.nTotalBlocks = 0
470 461
471 462 self.flagIsNewBlock = 0
472 463
473 464 self.setFile = None
474 465
475 466 self.dtype = None
476 467
477 468 self.path = None
478 469
479 470 self.noMoreFiles = 0
480 471
481 472 self.filename = None
482 473
483 474 self.basicHeaderObj = BasicHeader(LOCALTIME)
484 475
485 476 self.systemHeaderObj = SystemHeader()
486 477
487 478 self.radarControllerHeaderObj = RadarControllerHeader()
488 479
489 480 self.processingHeaderObj = ProcessingHeader()
490 481
491 482
492 483 def hasAllDataInBuffer(self):
493 484 return 1
494 485
495 486
496 487
497 488 def setBlockDimension(self):
498 489 """
499 490 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
500 491
501 492 Affected:
502 493 self.shape_spc_Buffer
503 494 self.shape_cspc_Buffer
504 495 self.shape_dc_Buffer
505 496
506 497 Return: None
507 498 """
508 499 self.shape_spc_Buffer = (self.dataOut.nChannels,
509 500 self.processingHeaderObj.nHeights,
510 501 self.processingHeaderObj.profilesPerBlock)
511 502
512 503 self.shape_cspc_Buffer = (self.dataOut.nPairs,
513 504 self.processingHeaderObj.nHeights,
514 505 self.processingHeaderObj.profilesPerBlock)
515 506
516 507 self.shape_dc_Buffer = (self.dataOut.nChannels,
517 508 self.processingHeaderObj.nHeights)
518 509
519 510
520 511 def writeBlock(self):
521 512 """
522 513 Escribe el buffer en el file designado
523 514
524 515
525 516 Affected:
526 517 self.data_spc
527 518 self.data_cspc
528 519 self.data_dc
529 520 self.flagIsNewFile
530 521 self.flagIsNewBlock
531 522 self.nTotalBlocks
532 523 self.nWriteBlocks
533 524
534 525 Return: None
535 526 """
536 527
537 528 spc = numpy.transpose( self.data_spc, (0,2,1) )
538 529 if not( self.processingHeaderObj.shif_fft ):
539 530 spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
540 531 data = spc.reshape((-1))
541 532 data = data.astype(self.dtype[0])
542 533 data.tofile(self.fp)
543 534
544 535 if self.data_cspc is not None:
545 536 data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )
546 537 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
547 538 if not( self.processingHeaderObj.shif_fft ):
548 539 cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
549 540 data['real'] = cspc.real
550 541 data['imag'] = cspc.imag
551 542 data = data.reshape((-1))
552 543 data.tofile(self.fp)
553 544
554 545
555 546 if self.data_dc is not None:
556 547 data = numpy.zeros( self.shape_dc_Buffer, self.dtype )
557 548 dc = self.data_dc
558 549 data['real'] = dc.real
559 550 data['imag'] = dc.imag
560 551 data = data.reshape((-1))
561 552 data.tofile(self.fp)
562 553
563 554 # self.data_spc.fill(0)
564 555 #
565 556 # if self.data_dc is not None:
566 557 # self.data_dc.fill(0)
567 558 #
568 559 # if self.data_cspc is not None:
569 560 # self.data_cspc.fill(0)
570 561
571 562
572 563 self.flagIsNewFile = 0
573 564 self.flagIsNewBlock = 1
574 565 self.nTotalBlocks += 1
575 566 self.nWriteBlocks += 1
576 567 self.blockIndex += 1
577 568
578 569 # print "[Writing] Block = %d04" %self.blockIndex
579 570
580 571 def putData(self):
581 572 """
582 573 Setea un bloque de datos y luego los escribe en un file
583 574
584 575
585 576 Affected:
586 577 self.data_spc
587 578 self.data_cspc
588 579 self.data_dc
589 580
590 581 Return:
591 582 0 : Si no hay data o no hay mas files que puedan escribirse
592 583 1 : Si se escribio la data de un bloque en un file
593 584 """
594 585
595 586 if self.dataOut.flagNoData:
596 587 return 0
597 588
598 589 self.flagIsNewBlock = 0
599 590
600 591 if self.dataOut.flagDiscontinuousBlock:
601 592 self.data_spc.fill(0)
602 593 self.data_cspc.fill(0)
603 594 self.data_dc.fill(0)
604 595 self.setNextFile()
605 596
606 597 if self.flagIsNewFile == 0:
607 598 self.setBasicHeader()
608 599
609 600 self.data_spc = self.dataOut.data_spc.copy()
610 601
611 602 if self.dataOut.data_cspc is not None:
612 603 self.data_cspc = self.dataOut.data_cspc.copy()
613 604
614 605 if self.dataOut.data_dc is not None:
615 606 self.data_dc = self.dataOut.data_dc.copy()
616 607
617 608 # #self.processingHeaderObj.dataBlocksPerFile)
618 609 if self.hasAllDataInBuffer():
619 610 # self.setFirstHeader()
620 611 self.writeNextBlock()
621 612
622 613 return 1
623 614
624 615
625 616 def __getBlockSize(self):
626 617 '''
627 618 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
628 619 '''
629 620
630 621 dtype_width = self.getDtypeWidth()
631 622
632 623 pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
633 624
634 625 pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
635 626 blocksize = (pts2write_SelfSpectra*dtype_width)
636 627
637 628 if self.dataOut.data_cspc is not None:
638 629 pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
639 630 blocksize += (pts2write_CrossSpectra*dtype_width*2)
640 631
641 632 if self.dataOut.data_dc is not None:
642 633 pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
643 634 blocksize += (pts2write_DCchannels*dtype_width*2)
644 635
645 636 # blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
646 637
647 638 return blocksize
648 639
649 640 def setFirstHeader(self):
650 641
651 642 """
652 643 Obtiene una copia del First Header
653 644
654 645 Affected:
655 646 self.systemHeaderObj
656 647 self.radarControllerHeaderObj
657 648 self.dtype
658 649
659 650 Return:
660 651 None
661 652 """
662 653
663 654 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
664 655 self.systemHeaderObj.nChannels = self.dataOut.nChannels
665 656 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
666 657
667 658 self.processingHeaderObj.dtype = 1 # Spectra
668 659 self.processingHeaderObj.blockSize = self.__getBlockSize()
669 660 self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
670 661 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
671 662 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
672 663 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
673 664 self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
674 665 self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
675 666 self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
676 667
677 668
678 669 if self.processingHeaderObj.totalSpectra > 0:
679 670 channelList = []
680 671 for channel in range(self.dataOut.nChannels):
681 672 channelList.append(channel)
682 673 channelList.append(channel)
683 674
684 675 pairsList = []
685 676 if self.dataOut.nPairs > 0:
686 677 for pair in self.dataOut.pairsList:
687 678 pairsList.append(pair[0])
688 679 pairsList.append(pair[1])
689 680
690 681 spectraComb = channelList + pairsList
691 682 spectraComb = numpy.array(spectraComb, dtype="u1")
692 683 self.processingHeaderObj.spectraComb = spectraComb
693 684
694 685 if self.dataOut.code is not None:
695 686 self.processingHeaderObj.code = self.dataOut.code
696 687 self.processingHeaderObj.nCode = self.dataOut.nCode
697 688 self.processingHeaderObj.nBaud = self.dataOut.nBaud
698 689
699 690 if self.processingHeaderObj.nWindows != 0:
700 691 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
701 692 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
702 693 self.processingHeaderObj.nHeights = self.dataOut.nHeights
703 694 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
704 695
705 696 self.processingHeaderObj.processFlags = self.getProcessFlags()
706 697
707 698 self.setBasicHeader()
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,604 +1,607
1 1 '''
2 2 @author: Juan C. Espinoza
3 3 '''
4 4
5 5 import time
6 6 import json
7 7 import numpy
8 8 import paho.mqtt.client as mqtt
9 9 import zmq
10 10 import datetime
11 11 from zmq.utils.monitor import recv_monitor_message
12 12 from functools import wraps
13 13 from threading import Thread
14 14 from multiprocessing import Process
15 15
16 16 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
17 17 from schainpy.model.data.jrodata import JROData
18 18 from schainpy.utils import log
19 19
20 20 MAXNUMX = 100
21 21 MAXNUMY = 100
22 22
23 23 class PrettyFloat(float):
24 24 def __repr__(self):
25 25 return '%.2f' % self
26 26
27 27 def roundFloats(obj):
28 28 if isinstance(obj, list):
29 29 return map(roundFloats, obj)
30 30 elif isinstance(obj, float):
31 31 return round(obj, 2)
32 32
33 33 def decimate(z, MAXNUMY):
34 34 dy = int(len(z[0])/MAXNUMY) + 1
35 35
36 36 return z[::, ::dy]
37 37
38 38 class throttle(object):
39 39 '''
40 40 Decorator that prevents a function from being called more than once every
41 41 time period.
42 42 To create a function that cannot be called more than once a minute, but
43 43 will sleep until it can be called:
44 44 @throttle(minutes=1)
45 45 def foo():
46 46 pass
47 47
48 48 for i in range(10):
49 49 foo()
50 50 print "This function has run %s times." % i
51 51 '''
52 52
53 53 def __init__(self, seconds=0, minutes=0, hours=0):
54 54 self.throttle_period = datetime.timedelta(
55 55 seconds=seconds, minutes=minutes, hours=hours
56 56 )
57 57
58 58 self.time_of_last_call = datetime.datetime.min
59 59
60 60 def __call__(self, fn):
61 61 @wraps(fn)
62 62 def wrapper(*args, **kwargs):
63 63 now = datetime.datetime.now()
64 64 time_since_last_call = now - self.time_of_last_call
65 65 time_left = self.throttle_period - time_since_last_call
66 66
67 67 if time_left > datetime.timedelta(seconds=0):
68 68 return
69 69
70 70 self.time_of_last_call = datetime.datetime.now()
71 71 return fn(*args, **kwargs)
72 72
73 73 return wrapper
74 74
75 75 class Data(object):
76 76 '''
77 77 Object to hold data to be plotted
78 78 '''
79 79
80 80 def __init__(self, plottypes, throttle_value):
81 81 self.plottypes = plottypes
82 82 self.throttle = throttle_value
83 83 self.ended = False
84 84 self.__times = []
85 85
86 86 def __str__(self):
87 87 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
88 88 return 'Data[{}][{}]'.format(';'.join(dum), len(self.__times))
89 89
90 90 def __len__(self):
91 91 return len(self.__times)
92 92
93 93 def __getitem__(self, key):
94 94 if key not in self.data:
95 95 raise KeyError(log.error('Missing key: {}'.format(key)))
96 96
97 97 if 'spc' in key:
98 98 ret = self.data[key]
99 99 else:
100 100 ret = numpy.array([self.data[key][x] for x in self.times])
101 101 if ret.ndim > 1:
102 102 ret = numpy.swapaxes(ret, 0, 1)
103 103 return ret
104 104
105 105 def setup(self):
106 106 '''
107 107 Configure object
108 108 '''
109 109
110 110 self.ended = False
111 111 self.data = {}
112 112 self.__times = []
113 113 self.__heights = []
114 114 self.__all_heights = set()
115 115 for plot in self.plottypes:
116 if 'snr' in plot:
117 plot = 'snr'
116 118 self.data[plot] = {}
117 119
118 120 def shape(self, key):
119 121 '''
120 122 Get the shape of the one-element data for the given key
121 123 '''
122 124
123 125 if len(self.data[key]):
124 126 if 'spc' in key:
125 127 return self.data[key].shape
126 128 return self.data[key][self.__times[0]].shape
127 129 return (0,)
128 130
129 131 def update(self, dataOut):
130 132 '''
131 133 Update data object with new dataOut
132 134 '''
133 135
134 136 tm = dataOut.utctime
135 137 if tm in self.__times:
136 138 return
137 139
138 140 self.parameters = getattr(dataOut, 'parameters', [])
139 141 self.pairs = dataOut.pairsList
140 142 self.channels = dataOut.channelList
141 self.xrange = (dataOut.getFreqRange(1)/1000. , dataOut.getAcfRange(1) , dataOut.getVelRange(1))
142 143 self.interval = dataOut.getTimeInterval()
144 if 'spc' in self.plottypes or 'cspc' in self.plottypes:
145 self.xrange = (dataOut.getFreqRange(1)/1000. , dataOut.getAcfRange(1) , dataOut.getVelRange(1))
143 146 self.__heights.append(dataOut.heightList)
144 147 self.__all_heights.update(dataOut.heightList)
145 148 self.__times.append(tm)
146 149
147 150 for plot in self.plottypes:
148 151 if plot == 'spc':
149 152 z = dataOut.data_spc/dataOut.normFactor
150 153 self.data[plot] = 10*numpy.log10(z)
151 154 if plot == 'cspc':
152 155 self.data[plot] = dataOut.data_cspc
153 156 if plot == 'noise':
154 157 self.data[plot][tm] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
155 158 if plot == 'rti':
156 159 self.data[plot][tm] = dataOut.getPower()
157 160 if plot == 'snr_db':
158 161 self.data['snr'][tm] = dataOut.data_SNR
159 162 if plot == 'snr':
160 163 self.data[plot][tm] = 10*numpy.log10(dataOut.data_SNR)
161 164 if plot == 'dop':
162 165 self.data[plot][tm] = 10*numpy.log10(dataOut.data_DOP)
163 166 if plot == 'mean':
164 167 self.data[plot][tm] = dataOut.data_MEAN
165 168 if plot == 'std':
166 169 self.data[plot][tm] = dataOut.data_STD
167 170 if plot == 'coh':
168 171 self.data[plot][tm] = dataOut.getCoherence()
169 172 if plot == 'phase':
170 173 self.data[plot][tm] = dataOut.getCoherence(phase=True)
171 174 if plot == 'output':
172 175 self.data[plot][tm] = dataOut.data_output
173 176 if plot == 'param':
174 177 self.data[plot][tm] = dataOut.data_param
175 178
176 179 def normalize_heights(self):
177 180 '''
178 181 Ensure same-dimension of the data for different heighList
179 182 '''
180 183
181 184 H = numpy.array(list(self.__all_heights))
182 185 H.sort()
183 186 for key in self.data:
184 187 shape = self.shape(key)[:-1] + H.shape
185 188 for tm, obj in self.data[key].items():
186 189 h = self.__heights[self.__times.index(tm)]
187 190 if H.size == h.size:
188 191 continue
189 192 index = numpy.where(numpy.in1d(H, h))[0]
190 193 dummy = numpy.zeros(shape) + numpy.nan
191 194 if len(shape) == 2:
192 195 dummy[:, index] = obj
193 196 else:
194 197 dummy[index] = obj
195 198 self.data[key][tm] = dummy
196 199
197 200 self.__heights = [H for tm in self.__times]
198 201
199 202 def jsonify(self, decimate=False):
200 203 '''
201 204 Convert data to json
202 205 '''
203 206
204 207 ret = {}
205 208 tm = self.times[-1]
206 209
207 210 for key, value in self.data:
208 211 if key in ('spc', 'cspc'):
209 212 ret[key] = roundFloats(self.data[key].to_list())
210 213 else:
211 214 ret[key] = roundFloats(self.data[key][tm].to_list())
212 215
213 216 ret['timestamp'] = tm
214 217 ret['interval'] = self.interval
215 218
216 219 @property
217 220 def times(self):
218 221 '''
219 222 Return the list of times of the current data
220 223 '''
221 224
222 225 ret = numpy.array(self.__times)
223 226 ret.sort()
224 227 return ret
225 228
226 229 @property
227 230 def heights(self):
228 231 '''
229 232 Return the list of heights of the current data
230 233 '''
231 234
232 235 return numpy.array(self.__heights[-1])
233 236
234 237 class PublishData(Operation):
235 238 '''
236 239 Operation to send data over zmq.
237 240 '''
238 241
239 242 def __init__(self, **kwargs):
240 243 """Inicio."""
241 244 Operation.__init__(self, **kwargs)
242 245 self.isConfig = False
243 246 self.client = None
244 247 self.zeromq = None
245 248 self.mqtt = None
246 249
247 250 def on_disconnect(self, client, userdata, rc):
248 251 if rc != 0:
249 252 log.warning('Unexpected disconnection.')
250 253 self.connect()
251 254
252 255 def connect(self):
253 256 log.warning('trying to connect')
254 257 try:
255 258 self.client.connect(
256 259 host=self.host,
257 260 port=self.port,
258 261 keepalive=60*10,
259 262 bind_address='')
260 263 self.client.loop_start()
261 264 # self.client.publish(
262 265 # self.topic + 'SETUP',
263 266 # json.dumps(setup),
264 267 # retain=True
265 268 # )
266 269 except:
267 270 log.error('MQTT Conection error.')
268 271 self.client = False
269 272
270 273 def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, verbose=True, **kwargs):
271 274 self.counter = 0
272 275 self.topic = kwargs.get('topic', 'schain')
273 276 self.delay = kwargs.get('delay', 0)
274 277 self.plottype = kwargs.get('plottype', 'spectra')
275 278 self.host = kwargs.get('host', "10.10.10.82")
276 279 self.port = kwargs.get('port', 3000)
277 280 self.clientId = clientId
278 281 self.cnt = 0
279 282 self.zeromq = zeromq
280 283 self.mqtt = kwargs.get('plottype', 0)
281 284 self.client = None
282 285 self.verbose = verbose
283 286 setup = []
284 287 if mqtt is 1:
285 288 self.client = mqtt.Client(
286 289 client_id=self.clientId + self.topic + 'SCHAIN',
287 290 clean_session=True)
288 291 self.client.on_disconnect = self.on_disconnect
289 292 self.connect()
290 293 for plot in self.plottype:
291 294 setup.append({
292 295 'plot': plot,
293 296 'topic': self.topic + plot,
294 297 'title': getattr(self, plot + '_' + 'title', False),
295 298 'xlabel': getattr(self, plot + '_' + 'xlabel', False),
296 299 'ylabel': getattr(self, plot + '_' + 'ylabel', False),
297 300 'xrange': getattr(self, plot + '_' + 'xrange', False),
298 301 'yrange': getattr(self, plot + '_' + 'yrange', False),
299 302 'zrange': getattr(self, plot + '_' + 'zrange', False),
300 303 })
301 304 if zeromq is 1:
302 305 context = zmq.Context()
303 306 self.zmq_socket = context.socket(zmq.PUSH)
304 307 server = kwargs.get('server', 'zmq.pipe')
305 308
306 309 if 'tcp://' in server:
307 310 address = server
308 311 else:
309 312 address = 'ipc:///tmp/%s' % server
310 313
311 314 self.zmq_socket.connect(address)
312 315 time.sleep(1)
313 316
314 317
315 318 def publish_data(self):
316 319 self.dataOut.finished = False
317 320 if self.mqtt is 1:
318 321 yData = self.dataOut.heightList[:2].tolist()
319 322 if self.plottype == 'spectra':
320 323 data = getattr(self.dataOut, 'data_spc')
321 324 z = data/self.dataOut.normFactor
322 325 zdB = 10*numpy.log10(z)
323 326 xlen, ylen = zdB[0].shape
324 327 dx = int(xlen/MAXNUMX) + 1
325 328 dy = int(ylen/MAXNUMY) + 1
326 329 Z = [0 for i in self.dataOut.channelList]
327 330 for i in self.dataOut.channelList:
328 331 Z[i] = zdB[i][::dx, ::dy].tolist()
329 332 payload = {
330 333 'timestamp': self.dataOut.utctime,
331 334 'data': roundFloats(Z),
332 335 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
333 336 'interval': self.dataOut.getTimeInterval(),
334 337 'type': self.plottype,
335 338 'yData': yData
336 339 }
337 340
338 341 elif self.plottype in ('rti', 'power'):
339 342 data = getattr(self.dataOut, 'data_spc')
340 343 z = data/self.dataOut.normFactor
341 344 avg = numpy.average(z, axis=1)
342 345 avgdB = 10*numpy.log10(avg)
343 346 xlen, ylen = z[0].shape
344 347 dy = numpy.floor(ylen/self.__MAXNUMY) + 1
345 348 AVG = [0 for i in self.dataOut.channelList]
346 349 for i in self.dataOut.channelList:
347 350 AVG[i] = avgdB[i][::dy].tolist()
348 351 payload = {
349 352 'timestamp': self.dataOut.utctime,
350 353 'data': roundFloats(AVG),
351 354 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
352 355 'interval': self.dataOut.getTimeInterval(),
353 356 'type': self.plottype,
354 357 'yData': yData
355 358 }
356 359 elif self.plottype == 'noise':
357 360 noise = self.dataOut.getNoise()/self.dataOut.normFactor
358 361 noisedB = 10*numpy.log10(noise)
359 362 payload = {
360 363 'timestamp': self.dataOut.utctime,
361 364 'data': roundFloats(noisedB.reshape(-1, 1).tolist()),
362 365 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
363 366 'interval': self.dataOut.getTimeInterval(),
364 367 'type': self.plottype,
365 368 'yData': yData
366 369 }
367 370 elif self.plottype == 'snr':
368 371 data = getattr(self.dataOut, 'data_SNR')
369 372 avgdB = 10*numpy.log10(data)
370 373
371 374 ylen = data[0].size
372 375 dy = numpy.floor(ylen/self.__MAXNUMY) + 1
373 376 AVG = [0 for i in self.dataOut.channelList]
374 377 for i in self.dataOut.channelList:
375 378 AVG[i] = avgdB[i][::dy].tolist()
376 379 payload = {
377 380 'timestamp': self.dataOut.utctime,
378 381 'data': roundFloats(AVG),
379 382 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
380 383 'type': self.plottype,
381 384 'yData': yData
382 385 }
383 386 else:
384 387 print "Tipo de grafico invalido"
385 388 payload = {
386 389 'data': 'None',
387 390 'timestamp': 'None',
388 391 'type': None
389 392 }
390 393
391 394 self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0)
392 395
393 396 if self.zeromq is 1:
394 397 if self.verbose:
395 398 log.log(
396 399 '{} - {}'.format(self.dataOut.type, self.dataOut.datatime),
397 400 'Sending'
398 401 )
399 402 self.zmq_socket.send_pyobj(self.dataOut)
400 403
401 404 def run(self, dataOut, **kwargs):
402 405 self.dataOut = dataOut
403 406 if not self.isConfig:
404 407 self.setup(**kwargs)
405 408 self.isConfig = True
406 409
407 410 self.publish_data()
408 411 time.sleep(self.delay)
409 412
410 413 def close(self):
411 414 if self.zeromq is 1:
412 415 self.dataOut.finished = True
413 416 self.zmq_socket.send_pyobj(self.dataOut)
414 417 time.sleep(0.1)
415 418 self.zmq_socket.close()
416 419 if self.client:
417 420 self.client.loop_stop()
418 421 self.client.disconnect()
419 422
420 423
421 424 class ReceiverData(ProcessingUnit):
422 425
423 426 def __init__(self, **kwargs):
424 427
425 428 ProcessingUnit.__init__(self, **kwargs)
426 429
427 430 self.isConfig = False
428 431 server = kwargs.get('server', 'zmq.pipe')
429 432 if 'tcp://' in server:
430 433 address = server
431 434 else:
432 435 address = 'ipc:///tmp/%s' % server
433 436
434 437 self.address = address
435 438 self.dataOut = JROData()
436 439
437 440 def setup(self):
438 441
439 442 self.context = zmq.Context()
440 443 self.receiver = self.context.socket(zmq.PULL)
441 444 self.receiver.bind(self.address)
442 445 time.sleep(0.5)
443 446 log.success('ReceiverData from {}'.format(self.address))
444 447
445 448
446 449 def run(self):
447 450
448 451 if not self.isConfig:
449 452 self.setup()
450 453 self.isConfig = True
451 454
452 455 self.dataOut = self.receiver.recv_pyobj()
453 456 log.log('{} - {}'.format(self.dataOut.type,
454 457 self.dataOut.datatime.ctime(),),
455 458 'Receiving')
456 459
457 460
458 461 class PlotterReceiver(ProcessingUnit, Process):
459 462
460 463 throttle_value = 5
461 464
462 465 def __init__(self, **kwargs):
463 466
464 467 ProcessingUnit.__init__(self, **kwargs)
465 468 Process.__init__(self)
466 469 self.mp = False
467 470 self.isConfig = False
468 471 self.isWebConfig = False
469 472 self.connections = 0
470 473 server = kwargs.get('server', 'zmq.pipe')
471 474 plot_server = kwargs.get('plot_server', 'zmq.web')
472 475 if 'tcp://' in server:
473 476 address = server
474 477 else:
475 478 address = 'ipc:///tmp/%s' % server
476 479
477 480 if 'tcp://' in plot_server:
478 481 plot_address = plot_server
479 482 else:
480 483 plot_address = 'ipc:///tmp/%s' % plot_server
481 484
482 485 self.address = address
483 486 self.plot_address = plot_address
484 487 self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')]
485 488 self.realtime = kwargs.get('realtime', False)
486 489 self.throttle_value = kwargs.get('throttle', 5)
487 490 self.sendData = self.initThrottle(self.throttle_value)
488 491 self.dates = []
489 492 self.setup()
490 493
491 494 def setup(self):
492 495
493 496 self.data = Data(self.plottypes, self.throttle_value)
494 497 self.isConfig = True
495 498
496 499 def event_monitor(self, monitor):
497 500
498 501 events = {}
499 502
500 503 for name in dir(zmq):
501 504 if name.startswith('EVENT_'):
502 505 value = getattr(zmq, name)
503 506 events[value] = name
504 507
505 508 while monitor.poll():
506 509 evt = recv_monitor_message(monitor)
507 510 if evt['event'] == 32:
508 511 self.connections += 1
509 512 if evt['event'] == 512:
510 513 pass
511 514
512 515 evt.update({'description': events[evt['event']]})
513 516
514 517 if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
515 518 break
516 519 monitor.close()
517 520 print('event monitor thread done!')
518 521
519 522 def initThrottle(self, throttle_value):
520 523
521 524 @throttle(seconds=throttle_value)
522 525 def sendDataThrottled(fn_sender, data):
523 526 fn_sender(data)
524 527
525 528 return sendDataThrottled
526 529
527 530 def send(self, data):
528 531 log.success('Sending {}'.format(data), self.name)
529 532 self.sender.send_pyobj(data)
530 533
531 534 def run(self):
532 535
533 536 log.success(
534 537 'Starting from {}'.format(self.address),
535 538 self.name
536 539 )
537 540
538 541 self.context = zmq.Context()
539 542 self.receiver = self.context.socket(zmq.PULL)
540 543 self.receiver.bind(self.address)
541 544 monitor = self.receiver.get_monitor_socket()
542 545 self.sender = self.context.socket(zmq.PUB)
543 546 if self.realtime:
544 547 self.sender_web = self.context.socket(zmq.PUB)
545 548 self.sender_web.connect(self.plot_address)
546 549 time.sleep(1)
547 550
548 551 if 'server' in self.kwargs:
549 552 self.sender.bind("ipc:///tmp/{}.plots".format(self.kwargs['server']))
550 553 else:
551 554 self.sender.bind("ipc:///tmp/zmq.plots")
552 555
553 556 time.sleep(2)
554 557
555 558 t = Thread(target=self.event_monitor, args=(monitor,))
556 559 t.start()
557 560
558 561 while True:
559 562 dataOut = self.receiver.recv_pyobj()
560 563 dt = datetime.datetime.fromtimestamp(dataOut.utctime).date()
561 564 sended = False
562 565 if dt not in self.dates:
563 566 if self.data:
564 567 self.data.ended = True
565 568 self.send(self.data)
566 569 sended = True
567 570 self.data.setup()
568 571 self.dates.append(dt)
569 572
570 573 self.data.update(dataOut)
571 574
572 575 if dataOut.finished is True:
573 576 self.connections -= 1
574 577 if self.connections == 0 and dt in self.dates:
575 578 self.data.ended = True
576 579 self.send(self.data)
577 580 self.data.setup()
578 581 else:
579 582 if self.realtime:
580 583 self.send(self.data)
581 584 # self.sender_web.send_string(self.data.jsonify())
582 585 else:
583 586 if not sended:
584 587 self.sendData(self.send, self.data)
585 588
586 589 return
587 590
588 591 def sendToWeb(self):
589 592
590 593 if not self.isWebConfig:
591 594 context = zmq.Context()
592 595 sender_web_config = context.socket(zmq.PUB)
593 596 if 'tcp://' in self.plot_address:
594 597 dum, address, port = self.plot_address.split(':')
595 598 conf_address = '{}:{}:{}'.format(dum, address, int(port)+1)
596 599 else:
597 600 conf_address = self.plot_address + '.config'
598 601 sender_web_config.bind(conf_address)
599 602 time.sleep(1)
600 603 for kwargs in self.operationKwargs.values():
601 604 if 'plot' in kwargs:
602 605 log.success('[Sending] Config data to web for {}'.format(kwargs['code'].upper()))
603 606 sender_web_config.send_string(json.dumps(kwargs))
604 607 self.isWebConfig = True No newline at end of file
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now