##// END OF EJS Templates
schain
RSS
Fix SendToFTP operation for v3
Juan C. Espinoza -
r1209:53188d049ed7
parent child
Show More
Side by Side Unified
Context file:
r1209:53188d049ed7 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,747 +1,748
1 1 '''
2 2 New Plots Operations
3 3
4 4 @author: juan.espinoza@jro.igp.gob.pe
5 5 '''
6 6
7 7
8 8 import time
9 9 import datetime
10 10 import numpy
11 11
12 12 from schainpy.model.graphics.jroplot_base import Plot, plt
13 13 from schainpy.utils import log
14 14
15 15 EARTH_RADIUS = 6.3710e3
16 16
17 17
18 18 def ll2xy(lat1, lon1, lat2, lon2):
19 19
20 20 p = 0.017453292519943295
21 21 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
22 22 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
23 23 r = 12742 * numpy.arcsin(numpy.sqrt(a))
24 24 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
25 25 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
26 26 theta = -theta + numpy.pi/2
27 27 return r*numpy.cos(theta), r*numpy.sin(theta)
28 28
29 29
30 30 def km2deg(km):
31 31 '''
32 32 Convert distance in km to degrees
33 33 '''
34 34
35 35 return numpy.rad2deg(km/EARTH_RADIUS)
36 36
37 37
38 38 class SpectraPlot(Plot):
39 39 '''
40 40 Plot for Spectra data
41 41 '''
42 42
43 43 CODE = 'spc'
44 44 colormap = 'jro'
45 45
46 46 def setup(self):
47 47 self.nplots = len(self.data.channels)
48 48 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
49 49 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
50 50 self.width = 3.4 * self.ncols
51 51 self.height = 3 * self.nrows
52 52 self.cb_label = 'dB'
53 53 if self.showprofile:
54 54 self.width += 0.8 * self.ncols
55 55
56 56 self.ylabel = 'Range [km]'
57 57
58 58 def plot(self):
59 59 if self.xaxis == "frequency":
60 60 x = self.data.xrange[0]
61 61 self.xlabel = "Frequency (kHz)"
62 62 elif self.xaxis == "time":
63 63 x = self.data.xrange[1]
64 64 self.xlabel = "Time (ms)"
65 65 else:
66 66 x = self.data.xrange[2]
67 67 self.xlabel = "Velocity (m/s)"
68 68
69 69 if self.CODE == 'spc_moments':
70 70 x = self.data.xrange[2]
71 71 self.xlabel = "Velocity (m/s)"
72 72
73 73 self.titles = []
74 74
75 75 y = self.data.heights
76 76 self.y = y
77 77 z = self.data['spc']
78 78
79 79 for n, ax in enumerate(self.axes):
80 80 noise = self.data['noise'][n][-1]
81 81 if self.CODE == 'spc_moments':
82 82 mean = self.data['moments'][n, :, 1, :][-1]
83 83 if ax.firsttime:
84 84 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
85 85 self.xmin = self.xmin if self.xmin else -self.xmax
86 86 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
87 87 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
88 88 ax.plt = ax.pcolormesh(x, y, z[n].T,
89 89 vmin=self.zmin,
90 90 vmax=self.zmax,
91 91 cmap=plt.get_cmap(self.colormap)
92 92 )
93 93
94 94 if self.showprofile:
95 95 ax.plt_profile = self.pf_axes[n].plot(
96 96 self.data['rti'][n][-1], y)[0]
97 97 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
98 98 color="k", linestyle="dashed", lw=1)[0]
99 99 if self.CODE == 'spc_moments':
100 100 ax.plt_mean = ax.plot(mean, y, color='k')[0]
101 101 else:
102 102 ax.plt.set_array(z[n].T.ravel())
103 103 if self.showprofile:
104 104 ax.plt_profile.set_data(self.data['rti'][n][-1], y)
105 105 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
106 106 if self.CODE == 'spc_moments':
107 107 ax.plt_mean.set_data(mean, y)
108 108 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
109 109
110 110
111 111 class CrossSpectraPlot(Plot):
112 112
113 113 CODE = 'cspc'
114 114 colormap = 'jet'
115 115 zmin_coh = None
116 116 zmax_coh = None
117 117 zmin_phase = None
118 118 zmax_phase = None
119 119
120 120 def setup(self):
121 121
122 122 self.ncols = 4
123 123 self.nrows = len(self.data.pairs)
124 124 self.nplots = self.nrows * 4
125 125 self.width = 3.4 * self.ncols
126 126 self.height = 3 * self.nrows
127 127 self.ylabel = 'Range [km]'
128 128 self.showprofile = False
129 129
130 130 def plot(self):
131 131
132 132 if self.xaxis == "frequency":
133 133 x = self.data.xrange[0]
134 134 self.xlabel = "Frequency (kHz)"
135 135 elif self.xaxis == "time":
136 136 x = self.data.xrange[1]
137 137 self.xlabel = "Time (ms)"
138 138 else:
139 139 x = self.data.xrange[2]
140 140 self.xlabel = "Velocity (m/s)"
141 141
142 142 self.titles = []
143 143
144 144 y = self.data.heights
145 145 self.y = y
146 146 spc = self.data['spc']
147 147 cspc = self.data['cspc']
148 148
149 149 for n in range(self.nrows):
150 150 noise = self.data['noise'][n][-1]
151 151 pair = self.data.pairs[n]
152 152 ax = self.axes[4 * n]
153 153 spc0 = 10.*numpy.log10(spc[pair[0]]/self.data.factor)
154 154 if ax.firsttime:
155 155 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
156 156 self.xmin = self.xmin if self.xmin else -self.xmax
157 157 self.zmin = self.zmin if self.zmin else numpy.nanmin(spc)
158 158 self.zmax = self.zmax if self.zmax else numpy.nanmax(spc)
159 159 ax.plt = ax.pcolormesh(x , y , spc0.T,
160 160 vmin=self.zmin,
161 161 vmax=self.zmax,
162 162 cmap=plt.get_cmap(self.colormap)
163 163 )
164 164 else:
165 165 ax.plt.set_array(spc0.T.ravel())
166 166 self.titles.append('CH {}: {:3.2f}dB'.format(pair[0], noise))
167 167
168 168 ax = self.axes[4 * n + 1]
169 169 spc1 = 10.*numpy.log10(spc[pair[1]]/self.data.factor)
170 170 if ax.firsttime:
171 171 ax.plt = ax.pcolormesh(x , y, spc1.T,
172 172 vmin=self.zmin,
173 173 vmax=self.zmax,
174 174 cmap=plt.get_cmap(self.colormap)
175 175 )
176 176 else:
177 177 ax.plt.set_array(spc1.T.ravel())
178 178 self.titles.append('CH {}: {:3.2f}dB'.format(pair[1], noise))
179 179
180 180 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
181 181 coh = numpy.abs(out)
182 182 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
183 183
184 184 ax = self.axes[4 * n + 2]
185 185 if ax.firsttime:
186 186 ax.plt = ax.pcolormesh(x, y, coh.T,
187 187 vmin=0,
188 188 vmax=1,
189 189 cmap=plt.get_cmap(self.colormap_coh)
190 190 )
191 191 else:
192 192 ax.plt.set_array(coh.T.ravel())
193 193 self.titles.append(
194 194 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
195 195
196 196 ax = self.axes[4 * n + 3]
197 197 if ax.firsttime:
198 198 ax.plt = ax.pcolormesh(x, y, phase.T,
199 199 vmin=-180,
200 200 vmax=180,
201 201 cmap=plt.get_cmap(self.colormap_phase)
202 202 )
203 203 else:
204 204 ax.plt.set_array(phase.T.ravel())
205 205 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
206 206
207 207
208 208 class SpectralMomentsPlot(SpectraPlot):
209 209 '''
210 210 Plot for Spectral Moments
211 211 '''
212 212 CODE = 'spc_moments'
213 213 colormap = 'jro'
214 214
215 215
216 216 class RTIPlot(Plot):
217 217 '''
218 218 Plot for RTI data
219 219 '''
220 220
221 221 CODE = 'rti'
222 222 colormap = 'jro'
223 223
224 224 def setup(self):
225 225 self.xaxis = 'time'
226 226 self.ncols = 1
227 227 self.nrows = len(self.data.channels)
228 228 self.nplots = len(self.data.channels)
229 229 self.ylabel = 'Range [km]'
230 230 self.cb_label = 'dB'
231 231 self.titles = ['{} Channel {}'.format(
232 232 self.CODE.upper(), x) for x in range(self.nrows)]
233 233
234 234 def plot(self):
235 235 self.x = self.data.times
236 236 self.y = self.data.heights
237 237 self.z = self.data[self.CODE]
238 238 self.z = numpy.ma.masked_invalid(self.z)
239 239
240 240 if self.decimation is None:
241 241 x, y, z = self.fill_gaps(self.x, self.y, self.z)
242 242 else:
243 243 x, y, z = self.fill_gaps(*self.decimate())
244 244
245 245 for n, ax in enumerate(self.axes):
246 246 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
247 247 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
248 248 if ax.firsttime:
249 249 ax.plt = ax.pcolormesh(x, y, z[n].T,
250 250 vmin=self.zmin,
251 251 vmax=self.zmax,
252 252 cmap=plt.get_cmap(self.colormap)
253 253 )
254 254 if self.showprofile:
255 255 ax.plot_profile = self.pf_axes[n].plot(
256 256 self.data['rti'][n][-1], self.y)[0]
257 257 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y,
258 258 color="k", linestyle="dashed", lw=1)[0]
259 259 else:
260 260 ax.collections.remove(ax.collections[0])
261 261 ax.plt = ax.pcolormesh(x, y, z[n].T,
262 262 vmin=self.zmin,
263 263 vmax=self.zmax,
264 264 cmap=plt.get_cmap(self.colormap)
265 265 )
266 266 if self.showprofile:
267 267 ax.plot_profile.set_data(self.data['rti'][n][-1], self.y)
268 268 ax.plot_noise.set_data(numpy.repeat(
269 269 self.data['noise'][n][-1], len(self.y)), self.y)
270 270
271 271
272 272 class CoherencePlot(RTIPlot):
273 273 '''
274 274 Plot for Coherence data
275 275 '''
276 276
277 277 CODE = 'coh'
278 278
279 279 def setup(self):
280 280 self.xaxis = 'time'
281 281 self.ncols = 1
282 282 self.nrows = len(self.data.pairs)
283 283 self.nplots = len(self.data.pairs)
284 284 self.ylabel = 'Range [km]'
285 285 if self.CODE == 'coh':
286 286 self.cb_label = ''
287 287 self.titles = [
288 288 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
289 289 else:
290 290 self.cb_label = 'Degrees'
291 291 self.titles = [
292 292 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
293 293
294 294
295 295 class PhasePlot(CoherencePlot):
296 296 '''
297 297 Plot for Phase map data
298 298 '''
299 299
300 300 CODE = 'phase'
301 301 colormap = 'seismic'
302 302
303 303
304 304 class NoisePlot(Plot):
305 305 '''
306 306 Plot for noise
307 307 '''
308 308
309 309 CODE = 'noise'
310 310
311 311 def setup(self):
312 312 self.xaxis = 'time'
313 313 self.ncols = 1
314 314 self.nrows = 1
315 315 self.nplots = 1
316 316 self.ylabel = 'Intensity [dB]'
317 317 self.titles = ['Noise']
318 318 self.colorbar = False
319 319
320 320 def plot(self):
321 321
322 322 x = self.data.times
323 323 xmin = self.data.min_time
324 324 xmax = xmin + self.xrange * 60 * 60
325 325 Y = self.data[self.CODE]
326 326
327 327 if self.axes[0].firsttime:
328 328 for ch in self.data.channels:
329 329 y = Y[ch]
330 330 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
331 331 plt.legend()
332 332 else:
333 333 for ch in self.data.channels:
334 334 y = Y[ch]
335 335 self.axes[0].lines[ch].set_data(x, y)
336 336
337 337 self.ymin = numpy.nanmin(Y) - 5
338 338 self.ymax = numpy.nanmax(Y) + 5
339 339
340 340
341 341 class SnrPlot(RTIPlot):
342 342 '''
343 343 Plot for SNR Data
344 344 '''
345 345
346 346 CODE = 'snr'
347 347 colormap = 'jet'
348 348
349 349
350 350 class DopplerPlot(RTIPlot):
351 351 '''
352 352 Plot for DOPPLER Data
353 353 '''
354 354
355 355 CODE = 'dop'
356 356 colormap = 'jet'
357 357
358 358
359 359 class SkyMapPlot(Plot):
360 360 '''
361 361 Plot for meteors detection data
362 362 '''
363 363
364 364 CODE = 'param'
365 365
366 366 def setup(self):
367 367
368 368 self.ncols = 1
369 369 self.nrows = 1
370 370 self.width = 7.2
371 371 self.height = 7.2
372 372 self.nplots = 1
373 373 self.xlabel = 'Zonal Zenith Angle (deg)'
374 374 self.ylabel = 'Meridional Zenith Angle (deg)'
375 375 self.polar = True
376 376 self.ymin = -180
377 377 self.ymax = 180
378 378 self.colorbar = False
379 379
380 380 def plot(self):
381 381
382 382 arrayParameters = numpy.concatenate(self.data['param'])
383 383 error = arrayParameters[:, -1]
384 384 indValid = numpy.where(error == 0)[0]
385 385 finalMeteor = arrayParameters[indValid, :]
386 386 finalAzimuth = finalMeteor[:, 3]
387 387 finalZenith = finalMeteor[:, 4]
388 388
389 389 x = finalAzimuth * numpy.pi / 180
390 390 y = finalZenith
391 391
392 392 ax = self.axes[0]
393 393
394 394 if ax.firsttime:
395 395 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
396 396 else:
397 397 ax.plot.set_data(x, y)
398 398
399 399 dt1 = self.getDateTime(self.data.min_time).strftime('%y/%m/%d %H:%M:%S')
400 400 dt2 = self.getDateTime(self.data.max_time).strftime('%y/%m/%d %H:%M:%S')
401 401 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
402 402 dt2,
403 403 len(x))
404 404 self.titles[0] = title
405 405
406 406
407 407 class ParametersPlot(RTIPlot):
408 408 '''
409 409 Plot for data_param object
410 410 '''
411 411
412 412 CODE = 'param'
413 413 colormap = 'seismic'
414 414
415 415 def setup(self):
416 416 self.xaxis = 'time'
417 417 self.ncols = 1
418 418 self.nrows = self.data.shape(self.CODE)[0]
419 419 self.nplots = self.nrows
420 420 if self.showSNR:
421 421 self.nrows += 1
422 422 self.nplots += 1
423 423
424 424 self.ylabel = 'Height [km]'
425 425 if not self.titles:
426 426 self.titles = self.data.parameters \
427 427 if self.data.parameters else ['Param {}'.format(x) for x in range(self.nrows)]
428 428 if self.showSNR:
429 429 self.titles.append('SNR')
430 430
431 431 def plot(self):
432 432 self.data.normalize_heights()
433 433 self.x = self.data.times
434 434 self.y = self.data.heights
435 435 if self.showSNR:
436 436 self.z = numpy.concatenate(
437 437 (self.data[self.CODE], self.data['snr'])
438 438 )
439 439 else:
440 440 self.z = self.data[self.CODE]
441 441
442 442 self.z = numpy.ma.masked_invalid(self.z)
443 443
444 444 if self.decimation is None:
445 445 x, y, z = self.fill_gaps(self.x, self.y, self.z)
446 446 else:
447 447 x, y, z = self.fill_gaps(*self.decimate())
448 448
449 449 for n, ax in enumerate(self.axes):
450 450
451 451 self.zmax = self.zmax if self.zmax is not None else numpy.max(
452 452 self.z[n])
453 453 self.zmin = self.zmin if self.zmin is not None else numpy.min(
454 454 self.z[n])
455 455
456 456 if ax.firsttime:
457 457 if self.zlimits is not None:
458 458 self.zmin, self.zmax = self.zlimits[n]
459 459
460 460 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
461 461 vmin=self.zmin,
462 462 vmax=self.zmax,
463 463 cmap=self.cmaps[n]
464 464 )
465 465 else:
466 466 if self.zlimits is not None:
467 467 self.zmin, self.zmax = self.zlimits[n]
468 468 ax.collections.remove(ax.collections[0])
469 469 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
470 470 vmin=self.zmin,
471 471 vmax=self.zmax,
472 472 cmap=self.cmaps[n]
473 473 )
474 474
475 475
476 476 class OutputPlot(ParametersPlot):
477 477 '''
478 478 Plot data_output object
479 479 '''
480 480
481 481 CODE = 'output'
482 482 colormap = 'seismic'
483 483
484 484
485 485 class PolarMapPlot(Plot):
486 486 '''
487 487 Plot for weather radar
488 488 '''
489 489
490 490 CODE = 'param'
491 491 colormap = 'seismic'
492 492
493 493 def setup(self):
494 494 self.ncols = 1
495 495 self.nrows = 1
496 496 self.width = 9
497 497 self.height = 8
498 498 self.mode = self.data.meta['mode']
499 499 if self.channels is not None:
500 500 self.nplots = len(self.channels)
501 501 self.nrows = len(self.channels)
502 502 else:
503 503 self.nplots = self.data.shape(self.CODE)[0]
504 504 self.nrows = self.nplots
505 505 self.channels = list(range(self.nplots))
506 506 if self.mode == 'E':
507 507 self.xlabel = 'Longitude'
508 508 self.ylabel = 'Latitude'
509 509 else:
510 510 self.xlabel = 'Range (km)'
511 511 self.ylabel = 'Height (km)'
512 512 self.bgcolor = 'white'
513 513 self.cb_labels = self.data.meta['units']
514 514 self.lat = self.data.meta['latitude']
515 515 self.lon = self.data.meta['longitude']
516 516 self.xmin, self.xmax = float(
517 517 km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
518 518 self.ymin, self.ymax = float(
519 519 km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
520 520 # self.polar = True
521 521
522 522 def plot(self):
523 523
524 524 for n, ax in enumerate(self.axes):
525 525 data = self.data['param'][self.channels[n]]
526 526
527 527 zeniths = numpy.linspace(
528 528 0, self.data.meta['max_range'], data.shape[1])
529 529 if self.mode == 'E':
530 530 azimuths = -numpy.radians(self.data.heights)+numpy.pi/2
531 531 r, theta = numpy.meshgrid(zeniths, azimuths)
532 532 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(
533 533 theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
534 534 x = km2deg(x) + self.lon
535 535 y = km2deg(y) + self.lat
536 536 else:
537 537 azimuths = numpy.radians(self.data.heights)
538 538 r, theta = numpy.meshgrid(zeniths, azimuths)
539 539 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
540 540 self.y = zeniths
541 541
542 542 if ax.firsttime:
543 543 if self.zlimits is not None:
544 544 self.zmin, self.zmax = self.zlimits[n]
545 545 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
546 546 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
547 547 vmin=self.zmin,
548 548 vmax=self.zmax,
549 549 cmap=self.cmaps[n])
550 550 else:
551 551 if self.zlimits is not None:
552 552 self.zmin, self.zmax = self.zlimits[n]
553 553 ax.collections.remove(ax.collections[0])
554 554 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
555 555 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
556 556 vmin=self.zmin,
557 557 vmax=self.zmax,
558 558 cmap=self.cmaps[n])
559 559
560 560 if self.mode == 'A':
561 561 continue
562 562
563 563 # plot district names
564 564 f = open('/data/workspace/schain_scripts/distrito.csv')
565 565 for line in f:
566 566 label, lon, lat = [s.strip() for s in line.split(',') if s]
567 567 lat = float(lat)
568 568 lon = float(lon)
569 569 # ax.plot(lon, lat, '.b', ms=2)
570 570 ax.text(lon, lat, label.decode('utf8'), ha='center',
571 571 va='bottom', size='8', color='black')
572 572
573 573 # plot limites
574 574 limites = []
575 575 tmp = []
576 576 for line in open('/data/workspace/schain_scripts/lima.csv'):
577 577 if '#' in line:
578 578 if tmp:
579 579 limites.append(tmp)
580 580 tmp = []
581 581 continue
582 582 values = line.strip().split(',')
583 583 tmp.append((float(values[0]), float(values[1])))
584 584 for points in limites:
585 585 ax.add_patch(
586 586 Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
587 587
588 588 # plot Cuencas
589 589 for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
590 590 f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
591 591 values = [line.strip().split(',') for line in f]
592 592 points = [(float(s[0]), float(s[1])) for s in values]
593 593 ax.add_patch(Polygon(points, ec='b', fc='none'))
594 594
595 595 # plot grid
596 596 for r in (15, 30, 45, 60):
597 597 ax.add_artist(plt.Circle((self.lon, self.lat),
598 598 km2deg(r), color='0.6', fill=False, lw=0.2))
599 599 ax.text(
600 600 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
601 601 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
602 602 '{}km'.format(r),
603 603 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
604 604
605 605 if self.mode == 'E':
606 606 title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
607 607 label = 'E{:02d}'.format(int(self.data.meta['elevation']))
608 608 else:
609 609 title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
610 610 label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
611 611
612 612 self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
613 613 self.titles = ['{} {}'.format(
614 614 self.data.parameters[x], title) for x in self.channels]
615 615
616
616 617 class ScopePlot(Plot):
617 618
618 619 '''
619 620 Plot for Scope
620 621 '''
621 622
622 623 CODE = 'scope'
623 624
624 625 def setup(self):
625 626
626 627 self.xaxis = 'Range (Km)'
627 628 self.ncols = 1
628 629 self.nrows = 1
629 630 self.nplots = 1
630 631 self.ylabel = 'Intensity [dB]'
631 632 self.titles = ['Scope']
632 633 self.colorbar = False
633 634 colspan = 3
634 635 rowspan = 1
635 636
636 637 def plot_iq(self, x, y, channelIndexList, thisDatetime, wintitle):
637 638
638 639 yreal = y[channelIndexList,:].real
639 640 yimag = y[channelIndexList,:].imag
640 641 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y"))
641 642 self.xlabel = "Range (Km)"
642 643 self.ylabel = "Intensity - IQ"
643 644
644 645 self.y = yreal
645 646 self.x = x
646 647 self.xmin = min(x)
647 648 self.xmax = max(x)
648 649
649 650
650 651 self.titles[0] = title
651 652
652 653 for i,ax in enumerate(self.axes):
653 654 title = "Channel %d" %(i)
654 655 if ax.firsttime:
655 656 ax.plt_r = ax.plot(x, yreal[i,:], color='b')[0]
656 657 ax.plt_i = ax.plot(x, yimag[i,:], color='r')[0]
657 658 else:
658 659 #pass
659 660 ax.plt_r.set_data(x, yreal[i,:])
660 661 ax.plt_i.set_data(x, yimag[i,:])
661 662
662 663 def plot_power(self, x, y, channelIndexList, thisDatetime, wintitle):
663 664 y = y[channelIndexList,:] * numpy.conjugate(y[channelIndexList,:])
664 665 yreal = y.real
665 666 self.y = yreal
666 667 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y"))
667 668 self.xlabel = "Range (Km)"
668 669 self.ylabel = "Intensity"
669 670 self.xmin = min(x)
670 671 self.xmax = max(x)
671 672
672 673
673 674 self.titles[0] = title
674 675
675 676 for i,ax in enumerate(self.axes):
676 677 title = "Channel %d" %(i)
677 678
678 679 ychannel = yreal[i,:]
679 680
680 681 if ax.firsttime:
681 682 ax.plt_r = ax.plot(x, ychannel)[0]
682 683 else:
683 684 #pass
684 685 ax.plt_r.set_data(x, ychannel)
685 686
686 687
687 688 def plot(self):
688 689
689 690 if self.channels:
690 691 channels = self.channels
691 692 else:
692 693 channels = self.data.channels
693 694
694 695
695 696
696 697 thisDatetime = datetime.datetime.utcfromtimestamp(self.data.times[-1])
697 698
698 699 scope = self.data['scope']
699 700
700 701
701 702 if self.data.flagDataAsBlock:
702 703
703 704 for i in range(self.data.nProfiles):
704 705
705 706 wintitle1 = " [Profile = %d] " %i
706 707
707 708 if self.type == "power":
708 709 self.plot_power(self.data.heights,
709 710 scope[:,i,:],
710 711 channels,
711 712 thisDatetime,
712 713 wintitle1
713 714 )
714 715
715 716 if self.type == "iq":
716 717 self.plot_iq(self.data.heights,
717 718 scope[:,i,:],
718 719 channels,
719 720 thisDatetime,
720 721 wintitle1
721 722 )
722 723
723 724
724 725
725 726
726 727
727 728 else:
728 729 wintitle = " [Profile = %d] " %self.data.profileIndex
729 730
730 731 if self.type == "power":
731 732 self.plot_power(self.data.heights,
732 733 scope,
733 734 channels,
734 735 thisDatetime,
735 736 wintitle
736 737 )
737 738
738 739 if self.type == "iq":
739 740 self.plot_iq(self.data.heights,
740 741 scope,
741 742 channels,
742 743 thisDatetime,
743 744 wintitle
744 745 )
745 746
746 747
747 748 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,309 +1,326
1 1 '''
2 2 @author: Juan C. Espinoza
3 3 '''
4 4
5 5 import os
6 6 import glob
7 7 import time
8 8 import json
9 9 import numpy
10 10 import zmq
11 11 import datetime
12 12 import ftplib
13 13 from functools import wraps
14 14 from threading import Thread
15 15 from multiprocessing import Process
16 16
17 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
17 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit, MPDecorator
18 18 from schainpy.model.data.jrodata import JROData
19 19 from schainpy.utils import log
20 20
21 21 MAXNUMX = 500
22 22 MAXNUMY = 500
23 23
24 24 PLOT_CODES = {
25 25 'rti': 0, # Range time intensity (RTI).
26 26 'spc': 1, # Spectra (and Cross-spectra) information.
27 27 'cspc': 2, # Cross-Correlation information.
28 28 'coh': 3, # Coherence map.
29 29 'base': 4, # Base lines graphic.
30 30 'row': 5, # Row Spectra.
31 31 'total': 6, # Total Power.
32 32 'drift': 7, # Drifts graphics.
33 33 'height': 8, # Height profile.
34 34 'phase': 9, # Signal Phase.
35 35 'power': 16,
36 36 'noise': 17,
37 37 'beacon': 18,
38 38 'wind': 22,
39 39 'skymap': 23,
40 40 'Unknown': 24,
41 41 'V-E': 25, # PIP Velocity.
42 42 'Z-E': 26, # PIP Reflectivity.
43 43 'V-A': 27, # RHI Velocity.
44 44 'Z-A': 28, # RHI Reflectivity.
45 45 }
46 46
47 47 def get_plot_code(s):
48 48 label = s.split('_')[0]
49 49 codes = [key for key in PLOT_CODES if key in label]
50 50 if codes:
51 51 return PLOT_CODES[codes[0]]
52 52 else:
53 53 return 24
54 54
55 55 def decimate(z, MAXNUMY):
56 56 dy = int(len(z[0])/MAXNUMY) + 1
57 57
58 58 return z[::, ::dy]
59 59
60 60
61 61 class PublishData(Operation):
62 62 '''
63 63 Operation to send data over zmq.
64 64 '''
65 65
66 66 __attrs__ = ['host', 'port', 'delay', 'verbose']
67 67
68 def __init__(self, **kwargs):
69 """Inicio."""
70 Operation.__init__(self, **kwargs)
71 self.isConfig = False
72
73 68 def setup(self, server='zmq.pipe', delay=0, verbose=True, **kwargs):
74 69 self.counter = 0
75 70 self.delay = kwargs.get('delay', 0)
76 71 self.cnt = 0
77 72 self.verbose = verbose
78 setup = []
79 73 context = zmq.Context()
80 74 self.zmq_socket = context.socket(zmq.PUSH)
81 75 server = kwargs.get('server', 'zmq.pipe')
82 76
83 77 if 'tcp://' in server:
84 78 address = server
85 79 else:
86 80 address = 'ipc:///tmp/%s' % server
87 81
88 82 self.zmq_socket.connect(address)
89 83 time.sleep(1)
90 84
91 85
92 86 def publish_data(self):
93 87 self.dataOut.finished = False
94 88
95 89 if self.verbose:
96 90 log.log(
97 91 'Sending {} - {}'.format(self.dataOut.type, self.dataOut.datatime),
98 92 self.name
99 93 )
100 94 self.zmq_socket.send_pyobj(self.dataOut)
101 95
102 96 def run(self, dataOut, **kwargs):
103 97 self.dataOut = dataOut
104 98 if not self.isConfig:
105 99 self.setup(**kwargs)
106 100 self.isConfig = True
107 101
108 102 self.publish_data()
109 103 time.sleep(self.delay)
110 104
111 105 def close(self):
112 106
113 107 self.dataOut.finished = True
114 108 self.zmq_socket.send_pyobj(self.dataOut)
115 109 time.sleep(0.1)
116 110 self.zmq_socket.close()
117 111
118 112
119 113 class ReceiverData(ProcessingUnit):
120 114
121 115 __attrs__ = ['server']
122 116
123 117 def __init__(self, **kwargs):
124 118
125 119 ProcessingUnit.__init__(self, **kwargs)
126 120
127 121 self.isConfig = False
128 122 server = kwargs.get('server', 'zmq.pipe')
129 123 if 'tcp://' in server:
130 124 address = server
131 125 else:
132 126 address = 'ipc:///tmp/%s' % server
133 127
134 128 self.address = address
135 129 self.dataOut = JROData()
136 130
137 131 def setup(self):
138 132
139 133 self.context = zmq.Context()
140 134 self.receiver = self.context.socket(zmq.PULL)
141 135 self.receiver.bind(self.address)
142 136 time.sleep(0.5)
143 137 log.success('ReceiverData from {}'.format(self.address))
144 138
145 139
146 140 def run(self):
147 141
148 142 if not self.isConfig:
149 143 self.setup()
150 144 self.isConfig = True
151 145
152 146 self.dataOut = self.receiver.recv_pyobj()
153 147 log.log('{} - {}'.format(self.dataOut.type,
154 148 self.dataOut.datatime.ctime(),),
155 149 'Receiving')
156 150
157
158 class SendToFTP(Operation, Process):
151 @MPDecorator
152 class SendToFTP(Operation):
159 153
160 154 '''
161 155 Operation to send data over FTP.
156 patternX = 'local, remote, ext, period, exp_code, sub_exp_code'
162 157 '''
163 158
164 __attrs__ = ['server', 'username', 'password', 'patterns', 'timeout']
159 __attrs__ = ['server', 'username', 'password', 'timeout', 'patternX']
165 160
166 def __init__(self, **kwargs):
161 def __init__(self):
167 162 '''
168 patterns = [(local1, remote1, ext, delay, exp_code, sub_exp_code), ...]
169 163 '''
170 Operation.__init__(self, **kwargs)
171 Process.__init__(self)
172 self.server = kwargs.get('server')
173 self.username = kwargs.get('username')
174 self.password = kwargs.get('password')
175 self.patterns = kwargs.get('patterns')
176 self.timeout = kwargs.get('timeout', 30)
177 self.times = [time.time() for p in self.patterns]
178 self.latest = ['' for p in self.patterns]
179 self.mp = False
164 Operation.__init__(self)
180 165 self.ftp = None
166 self.ready = False
181 167
182 def setup(self):
168 def setup(self, server, username, password, timeout, **kwargs):
169 '''
170 '''
171
172 self.server = server
173 self.username = username
174 self.password = password
175 self.timeout = timeout
176 self.patterns = []
177 self.times = []
178 self.latest = []
179 for arg, value in kwargs.items():
180 if 'pattern' in arg:
181 self.patterns.append(value)
182 self.times.append(time.time())
183 self.latest.append('')
184
185 def connect(self):
186 '''
187 '''
183 188
184 189 log.log('Connecting to ftp://{}'.format(self.server), self.name)
185 190 try:
186 191 self.ftp = ftplib.FTP(self.server, timeout=self.timeout)
187 192 except ftplib.all_errors:
188 193 log.error('Server connection fail: {}'.format(self.server), self.name)
189 194 if self.ftp is not None:
190 195 self.ftp.close()
191 196 self.ftp = None
192 self.isConfig = False
197 self.ready = False
193 198 return
194 199
195 200 try:
196 201 self.ftp.login(self.username, self.password)
197 202 except ftplib.all_errors:
198 203 log.error('The given username y/o password are incorrect', self.name)
199 204 if self.ftp is not None:
200 205 self.ftp.close()
201 206 self.ftp = None
202 self.isConfig = False
207 self.ready = False
203 208 return
204 209
205 210 log.success('Connection success', self.name)
206 self.isConfig = True
211 self.ready = True
207 212 return
208 213
209 214 def check(self):
210 215
211 216 try:
212 217 self.ftp.voidcmd("NOOP")
213 218 except:
214 219 log.warning('Connection lost... trying to reconnect', self.name)
215 220 if self.ftp is not None:
216 221 self.ftp.close()
217 222 self.ftp = None
218 self.setup()
223 self.connect()
219 224
220 225 def find_files(self, path, ext):
221 226
222 227 files = glob.glob1(path, '*{}'.format(ext))
223 228 files.sort()
224 229 if files:
225 230 return files[-1]
226 231 return None
227 232
228 233 def getftpname(self, filename, exp_code, sub_exp_code):
229 234
230 235 thisDatetime = datetime.datetime.strptime(filename.split('_')[1], '%Y%m%d')
231 YEAR_STR = '%4.4d'%thisDatetime.timetuple().tm_year
232 DOY_STR = '%3.3d'%thisDatetime.timetuple().tm_yday
233 exp_code = '%3.3d'%exp_code
234 sub_exp_code = '%2.2d'%sub_exp_code
235 plot_code = '%2.2d'% get_plot_code(filename)
236 YEAR_STR = '%4.4d' % thisDatetime.timetuple().tm_year
237 DOY_STR = '%3.3d' % thisDatetime.timetuple().tm_yday
238 exp_code = '%3.3d' % exp_code
239 sub_exp_code = '%2.2d' % sub_exp_code
240 plot_code = '%2.2d' % get_plot_code(filename)
236 241 name = YEAR_STR + DOY_STR + '00' + exp_code + sub_exp_code + plot_code + '00.png'
237 242 return name
238 243
239 244 def upload(self, src, dst):
240 245
241 log.log('Uploading {} '.format(src), self.name, nl=False)
246 log.log('Uploading {} -> {} '.format(
247 src.split('/')[-1], dst.split('/')[-1]),
248 self.name,
249 nl=False
250 )
242 251
243 252 fp = open(src, 'rb')
244 253 command = 'STOR {}'.format(dst)
245 254
246 255 try:
247 256 self.ftp.storbinary(command, fp, blocksize=1024)
248 257 except Exception as e:
249 258 log.error('{}'.format(e), self.name)
250 259 if self.ftp is not None:
251 260 self.ftp.close()
252 261 self.ftp = None
253 262 return 0
254 263
255 264 try:
256 265 self.ftp.sendcmd('SITE CHMOD 755 {}'.format(dst))
257 266 except Exception as e:
258 267 log.error('{}'.format(e), self.name)
259 268 if self.ftp is not None:
260 269 self.ftp.close()
261 270 self.ftp = None
262 271 return 0
263 272
264 273 fp.close()
265 274 log.success('OK', tag='')
266 275 return 1
267 276
268 277 def send_files(self):
269 278
270 279 for x, pattern in enumerate(self.patterns):
271 local, remote, ext, delay, exp_code, sub_exp_code = pattern
272 if time.time()-self.times[x] >= delay:
273 srcname = self.find_files(local, ext)
274 src = os.path.join(local, srcname)
275 if os.path.getmtime(src) < time.time() - 30*60:
280 local, remote, ext, period, exp_code, sub_exp_code = pattern
281 if time.time()-self.times[x] >= int(period):
282 srcname = self.find_files(local, ext)
283 src = os.path.join(local, srcname)
284 if os.path.getmtime(src) < time.time() - 30*60:
285 log.warning('Skipping old file {}'.format(srcname))
276 286 continue
277 287
278 288 if srcname is None or srcname == self.latest[x]:
289 log.warning('File alreday uploaded {}'.format(srcname))
279 290 continue
280 291
281 292 if 'png' in ext:
282 dstname = self.getftpname(srcname, exp_code, sub_exp_code)
293 dstname = self.getftpname(srcname, int(exp_code), int(sub_exp_code))
283 294 else:
284 295 dstname = srcname
285 296
286 297 dst = os.path.join(remote, dstname)
287 298
288 299 if self.upload(src, dst):
289 300 self.times[x] = time.time()
290 301 self.latest[x] = srcname
291 302 else:
292 self.isConfig = False
303 self.ready = False
293 304 break
294 305
295 def run(self):
306 def run(self, dataOut, server, username, password, timeout=10, **kwargs):
296 307
297 while True:
298 if not self.isConfig:
299 self.setup()
300 if self.ftp is not None:
301 self.check()
302 self.send_files()
303 time.sleep(10)
308 if not self.isConfig:
309 self.setup(
310 server=server,
311 username=username,
312 password=password,
313 timeout=timeout,
314 **kwargs
315 )
316 self.isConfig = True
317 if not self.ready:
318 self.connect()
319 if self.ftp is not None:
320 self.check()
321 self.send_files()
304 322
305 def close():
323 def close(self):
306 324
307 325 if self.ftp is not None:
308 326 self.ftp.close()
309 self.terminate() No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now