##// END OF EJS Templates
schain
RSS
-timerange bug fix at Meteor Phase Plot...
Julio Valdez -
r610:4d46abf6970f
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1362 +1,1363
1 1 import os
2 2 import datetime
3 3 import numpy
4 4
5 5 from figure import Figure, isRealtime
6 6 from plotting_codes import *
7 7
8 8 class MomentsPlot(Figure):
9 9
10 10 isConfig = None
11 11 __nsubplots = None
12 12
13 13 WIDTHPROF = None
14 14 HEIGHTPROF = None
15 15 PREFIX = 'prm'
16 16
17 17 def __init__(self):
18 18
19 19 self.isConfig = False
20 20 self.__nsubplots = 1
21 21
22 22 self.WIDTH = 280
23 23 self.HEIGHT = 250
24 24 self.WIDTHPROF = 120
25 25 self.HEIGHTPROF = 0
26 26 self.counter_imagwr = 0
27 27
28 28 self.PLOT_CODE = MOMENTS_CODE
29 29
30 30 self.FTP_WEI = None
31 31 self.EXP_CODE = None
32 32 self.SUB_EXP_CODE = None
33 33 self.PLOT_POS = None
34 34
35 35 def getSubplots(self):
36 36
37 37 ncol = int(numpy.sqrt(self.nplots)+0.9)
38 38 nrow = int(self.nplots*1./ncol + 0.9)
39 39
40 40 return nrow, ncol
41 41
42 42 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
43 43
44 44 self.__showprofile = showprofile
45 45 self.nplots = nplots
46 46
47 47 ncolspan = 1
48 48 colspan = 1
49 49 if showprofile:
50 50 ncolspan = 3
51 51 colspan = 2
52 52 self.__nsubplots = 2
53 53
54 54 self.createFigure(id = id,
55 55 wintitle = wintitle,
56 56 widthplot = self.WIDTH + self.WIDTHPROF,
57 57 heightplot = self.HEIGHT + self.HEIGHTPROF,
58 58 show=show)
59 59
60 60 nrow, ncol = self.getSubplots()
61 61
62 62 counter = 0
63 63 for y in range(nrow):
64 64 for x in range(ncol):
65 65
66 66 if counter >= self.nplots:
67 67 break
68 68
69 69 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
70 70
71 71 if showprofile:
72 72 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
73 73
74 74 counter += 1
75 75
76 76 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
77 77 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
78 78 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
79 79 server=None, folder=None, username=None, password=None,
80 80 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
81 81
82 82 """
83 83
84 84 Input:
85 85 dataOut :
86 86 id :
87 87 wintitle :
88 88 channelList :
89 89 showProfile :
90 90 xmin : None,
91 91 xmax : None,
92 92 ymin : None,
93 93 ymax : None,
94 94 zmin : None,
95 95 zmax : None
96 96 """
97 97
98 98 if dataOut.flagNoData:
99 99 return None
100 100
101 101 if realtime:
102 102 if not(isRealtime(utcdatatime = dataOut.utctime)):
103 103 print 'Skipping this plot function'
104 104 return
105 105
106 106 if channelList == None:
107 107 channelIndexList = dataOut.channelIndexList
108 108 else:
109 109 channelIndexList = []
110 110 for channel in channelList:
111 111 if channel not in dataOut.channelList:
112 112 raise ValueError, "Channel %d is not in dataOut.channelList"
113 113 channelIndexList.append(dataOut.channelList.index(channel))
114 114
115 115 factor = dataOut.normFactor
116 116 x = dataOut.abscissaList
117 117 y = dataOut.heightList
118 118
119 119 z = dataOut.data_pre[channelIndexList,:,:]/factor
120 120 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
121 121 avg = numpy.average(z, axis=1)
122 122 noise = dataOut.noise/factor
123 123
124 124 zdB = 10*numpy.log10(z)
125 125 avgdB = 10*numpy.log10(avg)
126 126 noisedB = 10*numpy.log10(noise)
127 127
128 128 #thisDatetime = dataOut.datatime
129 129 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
130 130 title = wintitle + " Parameters"
131 131 xlabel = "Velocity (m/s)"
132 132 ylabel = "Range (Km)"
133 133
134 134 if not self.isConfig:
135 135
136 136 nplots = len(channelIndexList)
137 137
138 138 self.setup(id=id,
139 139 nplots=nplots,
140 140 wintitle=wintitle,
141 141 showprofile=showprofile,
142 142 show=show)
143 143
144 144 if xmin == None: xmin = numpy.nanmin(x)
145 145 if xmax == None: xmax = numpy.nanmax(x)
146 146 if ymin == None: ymin = numpy.nanmin(y)
147 147 if ymax == None: ymax = numpy.nanmax(y)
148 148 if zmin == None: zmin = numpy.nanmin(avgdB)*0.9
149 149 if zmax == None: zmax = numpy.nanmax(avgdB)*0.9
150 150
151 151 self.FTP_WEI = ftp_wei
152 152 self.EXP_CODE = exp_code
153 153 self.SUB_EXP_CODE = sub_exp_code
154 154 self.PLOT_POS = plot_pos
155 155
156 156 self.isConfig = True
157 157
158 158 self.setWinTitle(title)
159 159
160 160 for i in range(self.nplots):
161 161 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
162 162 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[i]+1, noisedB[i], str_datetime)
163 163 axes = self.axesList[i*self.__nsubplots]
164 164 axes.pcolor(x, y, zdB[i,:,:],
165 165 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
166 166 xlabel=xlabel, ylabel=ylabel, title=title,
167 167 ticksize=9, cblabel='')
168 168 #Mean Line
169 169 mean = dataOut.data_param[i, 1, :]
170 170 axes.addpline(mean, y, idline=0, color="black", linestyle="solid", lw=1)
171 171
172 172 if self.__showprofile:
173 173 axes = self.axesList[i*self.__nsubplots +1]
174 174 axes.pline(avgdB[i], y,
175 175 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
176 176 xlabel='dB', ylabel='', title='',
177 177 ytick_visible=False,
178 178 grid='x')
179 179
180 180 noiseline = numpy.repeat(noisedB[i], len(y))
181 181 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
182 182
183 183 self.draw()
184 184
185 185 self.save(figpath=figpath,
186 186 figfile=figfile,
187 187 save=save,
188 188 ftp=ftp,
189 189 wr_period=wr_period,
190 190 thisDatetime=thisDatetime)
191 191
192 192
193 193
194 194 class SkyMapPlot(Figure):
195 195
196 196 __isConfig = None
197 197 __nsubplots = None
198 198
199 199 WIDTHPROF = None
200 200 HEIGHTPROF = None
201 201 PREFIX = 'mmap'
202 202
203 203 def __init__(self):
204 204
205 205 self.__isConfig = False
206 206 self.__nsubplots = 1
207 207
208 208 # self.WIDTH = 280
209 209 # self.HEIGHT = 250
210 210 self.WIDTH = 600
211 211 self.HEIGHT = 600
212 212 self.WIDTHPROF = 120
213 213 self.HEIGHTPROF = 0
214 214 self.counter_imagwr = 0
215 215
216 216 self.PLOT_CODE = MSKYMAP_CODE
217 217
218 218 self.FTP_WEI = None
219 219 self.EXP_CODE = None
220 220 self.SUB_EXP_CODE = None
221 221 self.PLOT_POS = None
222 222
223 223 def getSubplots(self):
224 224
225 225 ncol = int(numpy.sqrt(self.nplots)+0.9)
226 226 nrow = int(self.nplots*1./ncol + 0.9)
227 227
228 228 return nrow, ncol
229 229
230 230 def setup(self, id, nplots, wintitle, showprofile=False, show=True):
231 231
232 232 self.__showprofile = showprofile
233 233 self.nplots = nplots
234 234
235 235 ncolspan = 1
236 236 colspan = 1
237 237
238 238 self.createFigure(id = id,
239 239 wintitle = wintitle,
240 240 widthplot = self.WIDTH, #+ self.WIDTHPROF,
241 241 heightplot = self.HEIGHT,# + self.HEIGHTPROF,
242 242 show=show)
243 243
244 244 nrow, ncol = 1,1
245 245 counter = 0
246 246 x = 0
247 247 y = 0
248 248 self.addAxes(1, 1, 0, 0, 1, 1, True)
249 249
250 250 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
251 251 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
252 252 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
253 253 server=None, folder=None, username=None, password=None,
254 254 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
255 255
256 256 """
257 257
258 258 Input:
259 259 dataOut :
260 260 id :
261 261 wintitle :
262 262 channelList :
263 263 showProfile :
264 264 xmin : None,
265 265 xmax : None,
266 266 ymin : None,
267 267 ymax : None,
268 268 zmin : None,
269 269 zmax : None
270 270 """
271 271
272 272 arrayParameters = dataOut.data_param[0,:]
273 273 error = arrayParameters[:,-1]
274 274 indValid = numpy.where(error == 0)[0]
275 275 finalMeteor = arrayParameters[indValid,:]
276 276 finalAzimuth = finalMeteor[:,4]
277 277 finalZenith = finalMeteor[:,5]
278 278
279 279 x = finalAzimuth*numpy.pi/180
280 280 y = finalZenith
281 281
282 282
283 283 #thisDatetime = dataOut.datatime
284 284 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
285 285 title = wintitle + " Parameters"
286 286 xlabel = "Zonal Zenith Angle (deg) "
287 287 ylabel = "Meridional Zenith Angle (deg)"
288 288
289 289 if not self.__isConfig:
290 290
291 291 nplots = 1
292 292
293 293 self.setup(id=id,
294 294 nplots=nplots,
295 295 wintitle=wintitle,
296 296 showprofile=showprofile,
297 297 show=show)
298 298
299 299 self.FTP_WEI = ftp_wei
300 300 self.EXP_CODE = exp_code
301 301 self.SUB_EXP_CODE = sub_exp_code
302 302 self.PLOT_POS = plot_pos
303 303 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
304 304 self.firstdate = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
305 305 self.__isConfig = True
306 306
307 307 self.setWinTitle(title)
308 308
309 309 i = 0
310 310 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
311 311
312 312 axes = self.axesList[i*self.__nsubplots]
313 313 nevents = axes.x_buffer.shape[0] + x.shape[0]
314 314 title = "Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n" %(self.firstdate,str_datetime,nevents)
315 315 axes.polar(x, y,
316 316 title=title, xlabel=xlabel, ylabel=ylabel,
317 317 ticksize=9, cblabel='')
318 318
319 319 self.draw()
320 320
321 321 self.save(figpath=figpath,
322 322 figfile=figfile,
323 323 save=save,
324 324 ftp=ftp,
325 325 wr_period=wr_period,
326 326 thisDatetime=thisDatetime)
327 327
328 328
329 329 class WindProfilerPlot(Figure):
330 330
331 331 __isConfig = None
332 332 __nsubplots = None
333 333
334 334 WIDTHPROF = None
335 335 HEIGHTPROF = None
336 336 PREFIX = 'wind'
337 337
338 338 def __init__(self):
339 339
340 340 self.timerange = None
341 341 self.__isConfig = False
342 342 self.__nsubplots = 1
343 343
344 344 self.WIDTH = 800
345 345 self.HEIGHT = 150
346 346 self.WIDTHPROF = 120
347 347 self.HEIGHTPROF = 0
348 348 self.counter_imagwr = 0
349 349
350 350 self.PLOT_CODE = WIND_CODE
351 351
352 352 self.FTP_WEI = None
353 353 self.EXP_CODE = None
354 354 self.SUB_EXP_CODE = None
355 355 self.PLOT_POS = None
356 356 self.tmin = None
357 357 self.tmax = None
358 358
359 359 self.xmin = None
360 360 self.xmax = None
361 361
362 362 self.figfile = None
363 363
364 364 def getSubplots(self):
365 365
366 366 ncol = 1
367 367 nrow = self.nplots
368 368
369 369 return nrow, ncol
370 370
371 371 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
372 372
373 373 self.__showprofile = showprofile
374 374 self.nplots = nplots
375 375
376 376 ncolspan = 1
377 377 colspan = 1
378 378
379 379 self.createFigure(id = id,
380 380 wintitle = wintitle,
381 381 widthplot = self.WIDTH + self.WIDTHPROF,
382 382 heightplot = self.HEIGHT + self.HEIGHTPROF,
383 383 show=show)
384 384
385 385 nrow, ncol = self.getSubplots()
386 386
387 387 counter = 0
388 388 for y in range(nrow):
389 389 if counter >= self.nplots:
390 390 break
391 391
392 392 self.addAxes(nrow, ncol*ncolspan, y, 0, colspan, 1)
393 393 counter += 1
394 394
395 395 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='False',
396 396 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
397 397 zmax_ver = None, zmin_ver = None, SNRmin = None, SNRmax = None,
398 398 timerange=None, SNRthresh = None,
399 399 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
400 400 server=None, folder=None, username=None, password=None,
401 401 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
402 402 """
403 403
404 404 Input:
405 405 dataOut :
406 406 id :
407 407 wintitle :
408 408 channelList :
409 409 showProfile :
410 410 xmin : None,
411 411 xmax : None,
412 412 ymin : None,
413 413 ymax : None,
414 414 zmin : None,
415 415 zmax : None
416 416 """
417 417
418 418 if channelList == None:
419 419 channelIndexList = dataOut.channelIndexList
420 420 else:
421 421 channelIndexList = []
422 422 for channel in channelList:
423 423 if channel not in dataOut.channelList:
424 424 raise ValueError, "Channel %d is not in dataOut.channelList"
425 425 channelIndexList.append(dataOut.channelList.index(channel))
426 426
427 427 # if timerange != None:
428 428 # self.timerange = timerange
429 429 #
430 430 # tmin = None
431 431 # tmax = None
432 432
433 433 x = dataOut.getTimeRange1()
434 434 # y = dataOut.heightList
435 435 y = dataOut.heightList
436 436
437 437 z = dataOut.data_output.copy()
438 438 nplots = z.shape[0] #Number of wind dimensions estimated
439 439 nplotsw = nplots
440 440
441 441 #If there is a SNR function defined
442 442 if dataOut.data_SNR != None:
443 443 nplots += 1
444 444 SNR = dataOut.data_SNR
445 445 SNRavg = numpy.average(SNR, axis=0)
446 446
447 447 SNRdB = 10*numpy.log10(SNR)
448 448 SNRavgdB = 10*numpy.log10(SNRavg)
449 449
450 450 if SNRthresh == None: SNRthresh = -5.0
451 451 ind = numpy.where(SNRavg < 10**(SNRthresh/10))[0]
452 452
453 453 for i in range(nplotsw):
454 454 z[i,ind] = numpy.nan
455 455
456 456
457 457 # showprofile = False
458 458 # thisDatetime = dataOut.datatime
459 459 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
460 460 title = wintitle + "Wind"
461 461 xlabel = ""
462 462 ylabel = "Range (Km)"
463 463
464 464 if not self.__isConfig:
465 465
466 466 self.setup(id=id,
467 467 nplots=nplots,
468 468 wintitle=wintitle,
469 469 showprofile=showprofile,
470 470 show=show)
471 471
472 472 if timerange != None:
473 473 self.timerange = timerange
474 474
475 475 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
476 476
477 477 if ymin == None: ymin = numpy.nanmin(y)
478 478 if ymax == None: ymax = numpy.nanmax(y)
479 479
480 480 if zmax == None: zmax = numpy.nanmax(abs(z[range(2),:]))
481 481 #if numpy.isnan(zmax): zmax = 50
482 482 if zmin == None: zmin = -zmax
483 483
484 484 if nplotsw == 3:
485 485 if zmax_ver == None: zmax_ver = numpy.nanmax(abs(z[2,:]))
486 486 if zmin_ver == None: zmin_ver = -zmax_ver
487 487
488 488 if dataOut.data_SNR != None:
489 489 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
490 490 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
491 491
492 492
493 493 self.FTP_WEI = ftp_wei
494 494 self.EXP_CODE = exp_code
495 495 self.SUB_EXP_CODE = sub_exp_code
496 496 self.PLOT_POS = plot_pos
497 497
498 498 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
499 499 self.__isConfig = True
500 500 self.figfile = figfile
501 501
502 502 self.setWinTitle(title)
503 503
504 504 if ((self.xmax - x[1]) < (x[1]-x[0])):
505 505 x[1] = self.xmax
506 506
507 507 strWind = ['Zonal', 'Meridional', 'Vertical']
508 508 strCb = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)']
509 509 zmaxVector = [zmax, zmax, zmax_ver]
510 510 zminVector = [zmin, zmin, zmin_ver]
511 511 windFactor = [1,1,100]
512 512
513 513 for i in range(nplotsw):
514 514
515 515 title = "%s Wind: %s" %(strWind[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
516 516 axes = self.axesList[i*self.__nsubplots]
517 517
518 518 z1 = z[i,:].reshape((1,-1))*windFactor[i]
519 519
520 520 axes.pcolorbuffer(x, y, z1,
521 521 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zminVector[i], zmax=zmaxVector[i],
522 522 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
523 523 ticksize=9, cblabel=strCb[i], cbsize="1%", colormap="RdBu_r" )
524 524
525 525 if dataOut.data_SNR != None:
526 526 i += 1
527 527 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
528 528 axes = self.axesList[i*self.__nsubplots]
529 529
530 530 SNRavgdB = SNRavgdB.reshape((1,-1))
531 531
532 532 axes.pcolorbuffer(x, y, SNRavgdB,
533 533 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
534 534 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
535 535 ticksize=9, cblabel='', cbsize="1%", colormap="jet")
536 536
537 537 self.draw()
538 538
539 539 if x[1] >= self.axesList[0].xmax:
540 540 self.counter_imagwr = wr_period
541 541 self.__isConfig = False
542 542 self.figfile = None
543 543
544 544 self.save(figpath=figpath,
545 545 figfile=figfile,
546 546 save=save,
547 547 ftp=ftp,
548 548 wr_period=wr_period,
549 549 thisDatetime=thisDatetime,
550 550 update_figfile=False)
551 551
552 552
553 553 class ParametersPlot(Figure):
554 554
555 555 __isConfig = None
556 556 __nsubplots = None
557 557
558 558 WIDTHPROF = None
559 559 HEIGHTPROF = None
560 560 PREFIX = 'prm'
561 561
562 562 def __init__(self):
563 563
564 564 self.timerange = 2*60*60
565 565 self.__isConfig = False
566 566 self.__nsubplots = 1
567 567
568 568 self.WIDTH = 800
569 569 self.HEIGHT = 150
570 570 self.WIDTHPROF = 120
571 571 self.HEIGHTPROF = 0
572 572 self.counter_imagwr = 0
573 573
574 574 self.PLOT_CODE = PARMS_CODE
575 575
576 576 self.FTP_WEI = None
577 577 self.EXP_CODE = None
578 578 self.SUB_EXP_CODE = None
579 579 self.PLOT_POS = None
580 580 self.tmin = None
581 581 self.tmax = None
582 582
583 583 self.xmin = None
584 584 self.xmax = None
585 585
586 586 self.figfile = None
587 587
588 588 def getSubplots(self):
589 589
590 590 ncol = 1
591 591 nrow = self.nplots
592 592
593 593 return nrow, ncol
594 594
595 595 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
596 596
597 597 self.__showprofile = showprofile
598 598 self.nplots = nplots
599 599
600 600 ncolspan = 1
601 601 colspan = 1
602 602
603 603 self.createFigure(id = id,
604 604 wintitle = wintitle,
605 605 widthplot = self.WIDTH + self.WIDTHPROF,
606 606 heightplot = self.HEIGHT + self.HEIGHTPROF,
607 607 show=show)
608 608
609 609 nrow, ncol = self.getSubplots()
610 610
611 611 counter = 0
612 612 for y in range(nrow):
613 613 for x in range(ncol):
614 614
615 615 if counter >= self.nplots:
616 616 break
617 617
618 618 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
619 619
620 620 if showprofile:
621 621 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
622 622
623 623 counter += 1
624 624
625 625 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
626 626 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,timerange=None,
627 627 parameterIndex = None, onlyPositive = False,
628 628 SNRthresh = -numpy.inf, SNR = True, SNRmin = None, SNRmax = None, onlySNR = False,
629 629 DOP = True,
630 630 zlabel = "", parameterName = "", parameterObject = "data_param",
631 631 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
632 632 server=None, folder=None, username=None, password=None,
633 633 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
634 634
635 635 """
636 636
637 637 Input:
638 638 dataOut :
639 639 id :
640 640 wintitle :
641 641 channelList :
642 642 showProfile :
643 643 xmin : None,
644 644 xmax : None,
645 645 ymin : None,
646 646 ymax : None,
647 647 zmin : None,
648 648 zmax : None
649 649 """
650 650
651 651 data_param = getattr(dataOut, parameterObject)
652 652
653 653 if channelList == None:
654 654 channelIndexList = numpy.arange(data_param.shape[0])
655 655 else:
656 656 channelIndexList = numpy.array(channelList)
657 657
658 658 nchan = len(channelIndexList) #Number of channels being plotted
659 659
660 660 if nchan < 1:
661 661 return
662 662
663 663 nGraphsByChannel = 0
664 664
665 665 if SNR:
666 666 nGraphsByChannel += 1
667 667 if DOP:
668 668 nGraphsByChannel += 1
669 669
670 670 if nGraphsByChannel < 1:
671 671 return
672 672
673 673 nplots = nGraphsByChannel*nchan
674 674
675 675 if timerange != None:
676 676 self.timerange = timerange
677 677
678 678 #tmin = None
679 679 #tmax = None
680 680 if parameterIndex == None:
681 681 parameterIndex = 1
682 682
683 683 x = dataOut.getTimeRange1()
684 684 y = dataOut.heightList
685 685 z = data_param[channelIndexList,parameterIndex,:].copy()
686 686
687 687 zRange = dataOut.abscissaList
688 688 # nChannels = z.shape[0] #Number of wind dimensions estimated
689 689 # thisDatetime = dataOut.datatime
690 690
691 691 if dataOut.data_SNR != None:
692 692 SNRarray = dataOut.data_SNR[channelIndexList,:]
693 693 SNRdB = 10*numpy.log10(SNRarray)
694 694 # SNRavgdB = 10*numpy.log10(SNRavg)
695 695 ind = numpy.where(SNRdB < 10**(SNRthresh/10))
696 696 z[ind] = numpy.nan
697 697
698 698 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
699 699 title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
700 700 xlabel = ""
701 701 ylabel = "Range (Km)"
702 702
703 703 if (SNR and not onlySNR): nplots = 2*nplots
704 704
705 705 if onlyPositive:
706 706 colormap = "jet"
707 707 zmin = 0
708 708 else: colormap = "RdBu_r"
709 709
710 710 if not self.__isConfig:
711 711
712 712 self.setup(id=id,
713 713 nplots=nplots,
714 714 wintitle=wintitle,
715 715 showprofile=showprofile,
716 716 show=show)
717 717
718 718 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
719 719
720 720 if ymin == None: ymin = numpy.nanmin(y)
721 721 if ymax == None: ymax = numpy.nanmax(y)
722 722 if zmin == None: zmin = numpy.nanmin(zRange)
723 723 if zmax == None: zmax = numpy.nanmax(zRange)
724 724
725 725 if SNR:
726 726 if SNRmin == None: SNRmin = numpy.nanmin(SNRdB)
727 727 if SNRmax == None: SNRmax = numpy.nanmax(SNRdB)
728 728
729 729 self.FTP_WEI = ftp_wei
730 730 self.EXP_CODE = exp_code
731 731 self.SUB_EXP_CODE = sub_exp_code
732 732 self.PLOT_POS = plot_pos
733 733
734 734 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
735 735 self.__isConfig = True
736 736 self.figfile = figfile
737 737
738 738 self.setWinTitle(title)
739 739
740 740 if ((self.xmax - x[1]) < (x[1]-x[0])):
741 741 x[1] = self.xmax
742 742
743 743 for i in range(nchan):
744 744
745 745 if (SNR and not onlySNR): j = 2*i
746 746 else: j = i
747 747
748 748 j = nGraphsByChannel*i
749 749
750 750 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
751 751 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
752 752
753 753 if not onlySNR:
754 754 axes = self.axesList[j*self.__nsubplots]
755 755 z1 = z[i,:].reshape((1,-1))
756 756 axes.pcolorbuffer(x, y, z1,
757 757 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
758 758 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
759 759 ticksize=9, cblabel=zlabel, cbsize="1%")
760 760
761 761 if DOP:
762 762 title = "%s Channel %d: %s" %(parameterName, channelIndexList[i]+1, thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
763 763
764 764 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
765 765 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
766 766 axes = self.axesList[j]
767 767 z1 = z[i,:].reshape((1,-1))
768 768 axes.pcolorbuffer(x, y, z1,
769 769 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
770 770 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
771 771 ticksize=9, cblabel=zlabel, cbsize="1%")
772 772
773 773 if SNR:
774 774 title = "Channel %d Signal Noise Ratio (SNR): %s" %(channelIndexList[i]+1, thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
775 775 axes = self.axesList[(j)*self.__nsubplots]
776 776 if not onlySNR:
777 777 axes = self.axesList[(j + 1)*self.__nsubplots]
778 778
779 779 axes = self.axesList[(j + nGraphsByChannel-1)]
780 780
781 781 z1 = SNRdB[i,:].reshape((1,-1))
782 782 axes.pcolorbuffer(x, y, z1,
783 783 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
784 784 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap="jet",
785 785 ticksize=9, cblabel=zlabel, cbsize="1%")
786 786
787 787
788 788
789 789 self.draw()
790 790
791 791 if x[1] >= self.axesList[0].xmax:
792 792 self.counter_imagwr = wr_period
793 793 self.__isConfig = False
794 794 self.figfile = None
795 795
796 796 self.save(figpath=figpath,
797 797 figfile=figfile,
798 798 save=save,
799 799 ftp=ftp,
800 800 wr_period=wr_period,
801 801 thisDatetime=thisDatetime,
802 802 update_figfile=False)
803 803
804 804 class SpectralFittingPlot(Figure):
805 805
806 806 __isConfig = None
807 807 __nsubplots = None
808 808
809 809 WIDTHPROF = None
810 810 HEIGHTPROF = None
811 811 PREFIX = 'prm'
812 812
813 813
814 814 N = None
815 815 ippSeconds = None
816 816
817 817 def __init__(self):
818 818 self.__isConfig = False
819 819 self.__nsubplots = 1
820 820
821 821 self.PLOT_CODE = SPECFIT_CODE
822 822
823 823 self.WIDTH = 450
824 824 self.HEIGHT = 250
825 825 self.WIDTHPROF = 0
826 826 self.HEIGHTPROF = 0
827 827
828 828 def getSubplots(self):
829 829
830 830 ncol = int(numpy.sqrt(self.nplots)+0.9)
831 831 nrow = int(self.nplots*1./ncol + 0.9)
832 832
833 833 return nrow, ncol
834 834
835 835 def setup(self, id, nplots, wintitle, showprofile=False, show=True):
836 836
837 837 showprofile = False
838 838 self.__showprofile = showprofile
839 839 self.nplots = nplots
840 840
841 841 ncolspan = 5
842 842 colspan = 4
843 843 if showprofile:
844 844 ncolspan = 5
845 845 colspan = 4
846 846 self.__nsubplots = 2
847 847
848 848 self.createFigure(id = id,
849 849 wintitle = wintitle,
850 850 widthplot = self.WIDTH + self.WIDTHPROF,
851 851 heightplot = self.HEIGHT + self.HEIGHTPROF,
852 852 show=show)
853 853
854 854 nrow, ncol = self.getSubplots()
855 855
856 856 counter = 0
857 857 for y in range(nrow):
858 858 for x in range(ncol):
859 859
860 860 if counter >= self.nplots:
861 861 break
862 862
863 863 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
864 864
865 865 if showprofile:
866 866 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
867 867
868 868 counter += 1
869 869
870 870 def run(self, dataOut, id, cutHeight=None, fit=False, wintitle="", channelList=None, showprofile=True,
871 871 xmin=None, xmax=None, ymin=None, ymax=None,
872 872 save=False, figpath='./', figfile=None, show=True):
873 873
874 874 """
875 875
876 876 Input:
877 877 dataOut :
878 878 id :
879 879 wintitle :
880 880 channelList :
881 881 showProfile :
882 882 xmin : None,
883 883 xmax : None,
884 884 zmin : None,
885 885 zmax : None
886 886 """
887 887
888 888 if cutHeight==None:
889 889 h=270
890 890 heightindex = numpy.abs(cutHeight - dataOut.heightList).argmin()
891 891 cutHeight = dataOut.heightList[heightindex]
892 892
893 893 factor = dataOut.normFactor
894 894 x = dataOut.abscissaList[:-1]
895 895 #y = dataOut.getHeiRange()
896 896
897 897 z = dataOut.data_pre[:,:,heightindex]/factor
898 898 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
899 899 avg = numpy.average(z, axis=1)
900 900 listChannels = z.shape[0]
901 901
902 902 #Reconstruct Function
903 903 if fit==True:
904 904 groupArray = dataOut.groupList
905 905 listChannels = groupArray.reshape((groupArray.size))
906 906 listChannels.sort()
907 907 spcFitLine = numpy.zeros(z.shape)
908 908 constants = dataOut.constants
909 909
910 910 nGroups = groupArray.shape[0]
911 911 nChannels = groupArray.shape[1]
912 912 nProfiles = z.shape[1]
913 913
914 914 for f in range(nGroups):
915 915 groupChann = groupArray[f,:]
916 916 p = dataOut.data_param[f,:,heightindex]
917 917 # p = numpy.array([ 89.343967,0.14036615,0.17086219,18.89835291,1.58388365,1.55099167])
918 918 fitLineAux = dataOut.library.modelFunction(p, constants)*nProfiles
919 919 fitLineAux = fitLineAux.reshape((nChannels,nProfiles))
920 920 spcFitLine[groupChann,:] = fitLineAux
921 921 # spcFitLine = spcFitLine/factor
922 922
923 923 z = z[listChannels,:]
924 924 spcFitLine = spcFitLine[listChannels,:]
925 925 spcFitLinedB = 10*numpy.log10(spcFitLine)
926 926
927 927 zdB = 10*numpy.log10(z)
928 928 #thisDatetime = dataOut.datatime
929 929 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
930 930 title = wintitle + " Doppler Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
931 931 xlabel = "Velocity (m/s)"
932 932 ylabel = "Spectrum"
933 933
934 934 if not self.__isConfig:
935 935
936 936 nplots = listChannels.size
937 937
938 938 self.setup(id=id,
939 939 nplots=nplots,
940 940 wintitle=wintitle,
941 941 showprofile=showprofile,
942 942 show=show)
943 943
944 944 if xmin == None: xmin = numpy.nanmin(x)
945 945 if xmax == None: xmax = numpy.nanmax(x)
946 946 if ymin == None: ymin = numpy.nanmin(zdB)
947 947 if ymax == None: ymax = numpy.nanmax(zdB)+2
948 948
949 949 self.__isConfig = True
950 950
951 951 self.setWinTitle(title)
952 952 for i in range(self.nplots):
953 953 # title = "Channel %d: %4.2fdB" %(dataOut.channelList[i]+1, noisedB[i])
954 954 title = "Height %4.1f km\nChannel %d:" %(cutHeight, listChannels[i]+1)
955 955 axes = self.axesList[i*self.__nsubplots]
956 956 if fit == False:
957 957 axes.pline(x, zdB[i,:],
958 958 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
959 959 xlabel=xlabel, ylabel=ylabel, title=title
960 960 )
961 961 if fit == True:
962 962 fitline=spcFitLinedB[i,:]
963 963 y=numpy.vstack([zdB[i,:],fitline] )
964 964 legendlabels=['Data','Fitting']
965 965 axes.pmultilineyaxis(x, y,
966 966 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
967 967 xlabel=xlabel, ylabel=ylabel, title=title,
968 968 legendlabels=legendlabels, marker=None,
969 969 linestyle='solid', grid='both')
970 970
971 971 self.draw()
972 972
973 973 self.save(figpath=figpath,
974 974 figfile=figfile,
975 975 save=save,
976 976 ftp=ftp,
977 977 wr_period=wr_period,
978 978 thisDatetime=thisDatetime)
979 979
980 980
981 981 class EWDriftsPlot(Figure):
982 982
983 983 __isConfig = None
984 984 __nsubplots = None
985 985
986 986 WIDTHPROF = None
987 987 HEIGHTPROF = None
988 988 PREFIX = 'drift'
989 989
990 990 def __init__(self):
991 991
992 992 self.timerange = 2*60*60
993 993 self.isConfig = False
994 994 self.__nsubplots = 1
995 995
996 996 self.WIDTH = 800
997 997 self.HEIGHT = 150
998 998 self.WIDTHPROF = 120
999 999 self.HEIGHTPROF = 0
1000 1000 self.counter_imagwr = 0
1001 1001
1002 1002 self.PLOT_CODE = EWDRIFT_CODE
1003 1003
1004 1004 self.FTP_WEI = None
1005 1005 self.EXP_CODE = None
1006 1006 self.SUB_EXP_CODE = None
1007 1007 self.PLOT_POS = None
1008 1008 self.tmin = None
1009 1009 self.tmax = None
1010 1010
1011 1011 self.xmin = None
1012 1012 self.xmax = None
1013 1013
1014 1014 self.figfile = None
1015 1015
1016 1016 def getSubplots(self):
1017 1017
1018 1018 ncol = 1
1019 1019 nrow = self.nplots
1020 1020
1021 1021 return nrow, ncol
1022 1022
1023 1023 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1024 1024
1025 1025 self.__showprofile = showprofile
1026 1026 self.nplots = nplots
1027 1027
1028 1028 ncolspan = 1
1029 1029 colspan = 1
1030 1030
1031 1031 self.createFigure(id = id,
1032 1032 wintitle = wintitle,
1033 1033 widthplot = self.WIDTH + self.WIDTHPROF,
1034 1034 heightplot = self.HEIGHT + self.HEIGHTPROF,
1035 1035 show=show)
1036 1036
1037 1037 nrow, ncol = self.getSubplots()
1038 1038
1039 1039 counter = 0
1040 1040 for y in range(nrow):
1041 1041 if counter >= self.nplots:
1042 1042 break
1043 1043
1044 1044 self.addAxes(nrow, ncol*ncolspan, y, 0, colspan, 1)
1045 1045 counter += 1
1046 1046
1047 1047 def run(self, dataOut, id, wintitle="", channelList=None,
1048 1048 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
1049 1049 zmaxVertical = None, zminVertical = None, zmaxZonal = None, zminZonal = None,
1050 1050 timerange=None, SNRthresh = -numpy.inf, SNRmin = None, SNRmax = None, SNR_1 = False,
1051 1051 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
1052 1052 server=None, folder=None, username=None, password=None,
1053 1053 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1054 1054 """
1055 1055
1056 1056 Input:
1057 1057 dataOut :
1058 1058 id :
1059 1059 wintitle :
1060 1060 channelList :
1061 1061 showProfile :
1062 1062 xmin : None,
1063 1063 xmax : None,
1064 1064 ymin : None,
1065 1065 ymax : None,
1066 1066 zmin : None,
1067 1067 zmax : None
1068 1068 """
1069 1069
1070 1070 if timerange != None:
1071 1071 self.timerange = timerange
1072 1072
1073 1073 tmin = None
1074 1074 tmax = None
1075 1075
1076 1076 x = dataOut.getTimeRange1()
1077 1077 # y = dataOut.heightList
1078 1078 y = dataOut.heightList
1079 1079
1080 1080 z = dataOut.data_output
1081 1081 nplots = z.shape[0] #Number of wind dimensions estimated
1082 1082 nplotsw = nplots
1083 1083
1084 1084 #If there is a SNR function defined
1085 1085 if dataOut.data_SNR != None:
1086 1086 nplots += 1
1087 1087 SNR = dataOut.data_SNR
1088 1088
1089 1089 if SNR_1:
1090 1090 SNR += 1
1091 1091
1092 1092 SNRavg = numpy.average(SNR, axis=0)
1093 1093
1094 1094 SNRdB = 10*numpy.log10(SNR)
1095 1095 SNRavgdB = 10*numpy.log10(SNRavg)
1096 1096
1097 1097 ind = numpy.where(SNRavg < 10**(SNRthresh/10))[0]
1098 1098
1099 1099 for i in range(nplotsw):
1100 1100 z[i,ind] = numpy.nan
1101 1101
1102 1102
1103 1103 showprofile = False
1104 1104 # thisDatetime = dataOut.datatime
1105 1105 thisDatetime = datetime.datetime.utcfromtimestamp(x[1])
1106 1106 title = wintitle + " EW Drifts"
1107 1107 xlabel = ""
1108 1108 ylabel = "Height (Km)"
1109 1109
1110 1110 if not self.__isConfig:
1111 1111
1112 1112 self.setup(id=id,
1113 1113 nplots=nplots,
1114 1114 wintitle=wintitle,
1115 1115 showprofile=showprofile,
1116 1116 show=show)
1117 1117
1118 1118 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1119 1119
1120 1120 if ymin == None: ymin = numpy.nanmin(y)
1121 1121 if ymax == None: ymax = numpy.nanmax(y)
1122 1122
1123 1123 if zmaxZonal == None: zmaxZonal = numpy.nanmax(abs(z[0,:]))
1124 1124 if zminZonal == None: zminZonal = -zmaxZonal
1125 1125 if zmaxVertical == None: zmaxVertical = numpy.nanmax(abs(z[1,:]))
1126 1126 if zminVertical == None: zminVertical = -zmaxVertical
1127 1127
1128 1128 if dataOut.data_SNR != None:
1129 1129 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
1130 1130 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
1131 1131
1132 1132 self.FTP_WEI = ftp_wei
1133 1133 self.EXP_CODE = exp_code
1134 1134 self.SUB_EXP_CODE = sub_exp_code
1135 1135 self.PLOT_POS = plot_pos
1136 1136
1137 1137 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1138 1138 self.__isConfig = True
1139 1139
1140 1140
1141 1141 self.setWinTitle(title)
1142 1142
1143 1143 if ((self.xmax - x[1]) < (x[1]-x[0])):
1144 1144 x[1] = self.xmax
1145 1145
1146 1146 strWind = ['Zonal','Vertical']
1147 1147 strCb = 'Velocity (m/s)'
1148 1148 zmaxVector = [zmaxZonal, zmaxVertical]
1149 1149 zminVector = [zminZonal, zminVertical]
1150 1150
1151 1151 for i in range(nplotsw):
1152 1152
1153 1153 title = "%s Drifts: %s" %(strWind[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1154 1154 axes = self.axesList[i*self.__nsubplots]
1155 1155
1156 1156 z1 = z[i,:].reshape((1,-1))
1157 1157
1158 1158 axes.pcolorbuffer(x, y, z1,
1159 1159 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zminVector[i], zmax=zmaxVector[i],
1160 1160 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
1161 1161 ticksize=9, cblabel=strCb, cbsize="1%", colormap="RdBu_r")
1162 1162
1163 1163 if dataOut.data_SNR != None:
1164 1164 i += 1
1165 1165 if SNR_1:
1166 1166 title = "Signal Noise Ratio + 1 (SNR+1): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1167 1167 else:
1168 1168 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1169 1169 axes = self.axesList[i*self.__nsubplots]
1170 1170 SNRavgdB = SNRavgdB.reshape((1,-1))
1171 1171
1172 1172 axes.pcolorbuffer(x, y, SNRavgdB,
1173 1173 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
1174 1174 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
1175 1175 ticksize=9, cblabel='', cbsize="1%", colormap="jet")
1176 1176
1177 1177 self.draw()
1178 1178
1179 1179 if x[1] >= self.axesList[0].xmax:
1180 1180 self.counter_imagwr = wr_period
1181 1181 self.__isConfig = False
1182 1182 self.figfile = None
1183 1183
1184 1184
1185 1185
1186 1186
1187 1187 class PhasePlot(Figure):
1188 1188
1189 1189 __isConfig = None
1190 1190 __nsubplots = None
1191 1191
1192 1192 PREFIX = 'mphase'
1193 1193
1194 1194 def __init__(self):
1195 1195
1196 1196 self.timerange = 24*60*60
1197 1197 self.__isConfig = False
1198 1198 self.__nsubplots = 1
1199 1199 self.counter_imagwr = 0
1200 1200 self.WIDTH = 600
1201 1201 self.HEIGHT = 300
1202 1202 self.WIDTHPROF = 120
1203 1203 self.HEIGHTPROF = 0
1204 1204 self.xdata = None
1205 1205 self.ydata = None
1206 1206
1207 1207 self.PLOT_CODE = MPHASE_CODE
1208 1208
1209 1209 self.FTP_WEI = None
1210 1210 self.EXP_CODE = None
1211 1211 self.SUB_EXP_CODE = None
1212 1212 self.PLOT_POS = None
1213 1213
1214 1214
1215 1215 self.filename_phase = None
1216 1216
1217 1217 self.figfile = None
1218 1218
1219 1219 def getSubplots(self):
1220 1220
1221 1221 ncol = 1
1222 1222 nrow = 1
1223 1223
1224 1224 return nrow, ncol
1225 1225
1226 1226 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1227 1227
1228 1228 self.__showprofile = showprofile
1229 1229 self.nplots = nplots
1230 1230
1231 1231 ncolspan = 7
1232 1232 colspan = 6
1233 1233 self.__nsubplots = 2
1234 1234
1235 1235 self.createFigure(id = id,
1236 1236 wintitle = wintitle,
1237 1237 widthplot = self.WIDTH+self.WIDTHPROF,
1238 1238 heightplot = self.HEIGHT+self.HEIGHTPROF,
1239 1239 show=show)
1240 1240
1241 1241 nrow, ncol = self.getSubplots()
1242 1242
1243 1243 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1244 1244
1245 1245
1246 1246 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
1247 1247 xmin=None, xmax=None, ymin=None, ymax=None,
1248 1248 timerange=None,
1249 1249 save=False, figpath='', figfile=None, show=True, ftp=False, wr_period=1,
1250 1250 server=None, folder=None, username=None, password=None,
1251 1251 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1252 1252
1253 1253
1254 if timerange != None:
1255 self.timerange = timerange
1256
1257 1254 tmin = None
1258 1255 tmax = None
1259 1256 x = dataOut.getTimeRange1()
1260 1257 y = dataOut.getHeiRange()
1261 1258
1262 1259
1263 1260 #thisDatetime = dataOut.datatime
1264 1261 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
1265 1262 title = wintitle + " Phase of Beacon Signal" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1266 1263 xlabel = "Local Time"
1267 1264 ylabel = "Phase"
1268 1265
1269 1266
1270 1267 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
1271 1268 phase_beacon = dataOut.data_output
1272 1269
1273 1270
1274 1271 if not self.__isConfig:
1275 1272
1276 1273 self.nplots = phase_beacon.size
1277 1274
1278 1275 self.setup(id=id,
1279 1276 nplots=self.nplots,
1280 1277 wintitle=wintitle,
1281 1278 showprofile=showprofile,
1282 1279 show=show)
1283 1280
1284 tmin, tmax = self.getTimeLim(x, xmin, xmax)
1281 if timerange != None:
1282 self.timerange = timerange
1283
1284 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1285
1285 1286 if ymin == None: ymin = numpy.nanmin(phase_beacon) - 10.0
1286 1287 if ymax == None: ymax = numpy.nanmax(phase_beacon) + 10.0
1287 1288
1288 1289 self.FTP_WEI = ftp_wei
1289 1290 self.EXP_CODE = exp_code
1290 1291 self.SUB_EXP_CODE = sub_exp_code
1291 1292 self.PLOT_POS = plot_pos
1292 1293
1293 1294 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1294 1295 self.__isConfig = True
1295 1296 self.figfile = figfile
1296 1297 self.xdata = numpy.array([])
1297 1298 self.ydata = numpy.array([])
1298 1299
1299 1300 #open file beacon phase
1300 1301 path = '%s%03d' %(self.PREFIX, self.id)
1301 1302 beacon_file = os.path.join(path,'%s.txt'%self.name)
1302 1303 self.filename_phase = os.path.join(figpath,beacon_file)
1303 1304 #self.save_phase(self.filename_phase)
1304 1305
1305 1306
1306 1307 #store data beacon phase
1307 1308 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
1308 1309
1309 1310 self.setWinTitle(title)
1310 1311
1311 1312
1312 1313 title = "Phase Offset %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1313 1314
1314 1315 legendlabels = ["phase %d"%(chan) for chan in numpy.arange(self.nplots)]
1315 1316
1316 1317 axes = self.axesList[0]
1317 1318
1318 1319 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1319 1320
1320 1321 if len(self.ydata)==0:
1321 1322 self.ydata = phase_beacon.reshape(-1,1)
1322 1323 else:
1323 1324 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
1324 1325
1325 1326
1326 1327 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1327 xmin=tmin, xmax=tmax, ymin=ymin, ymax=ymax,
1328 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1328 1329 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1329 1330 XAxisAsTime=True, grid='both'
1330 1331 )
1331 1332
1332 1333 self.draw()
1333 1334
1334 1335 if x[1] >= self.axesList[0].xmax:
1335 1336 self.counter_imagwr = wr_period
1336 1337 del self.xdata
1337 1338 del self.ydata
1338 1339 self.__isConfig = False
1339 1340
1340 1341 if self.figfile == None:
1341 1342 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
1342 1343 self.figfile = self.getFilename(name = str_datetime)
1343 1344
1344 1345 if figpath != '':
1345 1346 self.counter_imagwr += 1
1346 1347 if (self.counter_imagwr>=wr_period):
1347 1348 # store png plot to local folder
1348 1349 self.saveFigure(figpath, self.figfile)
1349 1350 # store png plot to FTP server according to RT-Web format
1350 1351 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
1351 1352 ftp_filename = os.path.join(figpath, name)
1352 1353 self.saveFigure(figpath, ftp_filename)
1353 1354 self.counter_imagwr = 0
1354 1355 self.figfile = None
1355 1356
1356 1357 self.save(figpath=figpath,
1357 1358 figfile=figfile,
1358 1359 save=save,
1359 1360 ftp=ftp,
1360 1361 wr_period=wr_period,
1361 1362 thisDatetime=thisDatetime,
1362 1363 update_figfile=False)
Show file before | Show file after | Hide comments
@@ -1,1005 +1,1005
1 1 import numpy
2 2 import time
3 3 import os
4 4 import h5py
5 5 import re
6 6 import tables
7 7
8 8 from schainpy.model.data.jrodata import *
9 9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 10 from schainpy.model.io.jroIO_base import *
11 11
12 12
13 13 class HDF5Reader(ProcessingUnit):
14 14
15 15 ext = ".hdf5"
16 16
17 17 optchar = "D"
18 18
19 19 timezone = None
20 20
21 21 secStart = None
22 22
23 23 secEnd = None
24 24
25 25 fileIndex = None
26 26
27 27 blockIndex = None
28 28
29 29 blocksPerFile = None
30 30
31 31 path = None
32 32
33 33 #List of Files
34 34
35 35 filenameList = None
36 36
37 37 datetimeList = None
38 38
39 39 #Hdf5 File
40 40
41 41 fpMetadata = None
42 42
43 43 pathMeta = None
44 44
45 45 listMetaname = None
46 46
47 47 listMeta = None
48 48
49 49 listDataname = None
50 50
51 51 listData = None
52 52
53 53 listShapes = None
54 54
55 55 fp = None
56 56
57 57 #dataOut reconstruction
58 58
59 59 dataOut = None
60 60
61 61 nRecords = None
62 62
63 63
64 64 def __init__(self):
65 65 self.dataOut = self.__createObjByDefault()
66 66 return
67 67
68 68 def __createObjByDefault(self):
69 69
70 70 dataObj = Parameters()
71 71
72 72 return dataObj
73 73
74 74 def setup(self,path=None,
75 75 startDate=None,
76 76 endDate=None,
77 77 startTime=datetime.time(0,0,0),
78 78 endTime=datetime.time(23,59,59),
79 79 walk=True,
80 80 timezone='ut',
81 81 all=0,
82 82 online=False,
83 83 ext=None):
84 84
85 85 if ext==None:
86 86 ext = self.ext
87 87 self.timezone = timezone
88 88 # self.all = all
89 89 # self.online = online
90 90 self.path = path
91 91
92 92 startDateTime = datetime.datetime.combine(startDate,startTime)
93 93 endDateTime = datetime.datetime.combine(endDate,endTime)
94 94 secStart = (startDateTime-datetime.datetime(1970,1,1)).total_seconds()
95 95 secEnd = (endDateTime-datetime.datetime(1970,1,1)).total_seconds()
96 96
97 97 self.secStart = secStart
98 98 self.secEnd = secEnd
99 99
100 100 if not(online):
101 101 #Busqueda de archivos offline
102 102 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, secStart, secEnd, walk)
103 103 else:
104 104 self.__searchFilesOnline(path, walk)
105 105
106 106 if not(self.filenameList):
107 107 print "There is no files into the folder: %s"%(path)
108 108 sys.exit(-1)
109 109
110 110 # self.__getExpParameters()
111 111
112 112 self.fileIndex = -1
113 113
114 114 self.__setNextFileOffline()
115 115
116 116 self.__readMetadata()
117 117
118 118 self.blockIndex = 0
119 119
120 120 return
121 121
122 122 def __searchFilesOffline(self,
123 123 path,
124 124 startDate,
125 125 endDate,
126 126 ext,
127 127 startTime=datetime.time(0,0,0),
128 128 endTime=datetime.time(23,59,59),
129 129 secStart = 0,
130 130 secEnd = numpy.inf,
131 131 walk=True):
132 132
133 133 # self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
134 134 #
135 135 # self.__checkPath()
136 136 #
137 137 # self.__findDataForDates()
138 138 #
139 139 # self.__selectDataForTimes()
140 140 #
141 141 # for i in range(len(self.filenameList)):
142 142 # print "%s" %(self.filenameList[i])
143 143
144 144 pathList = []
145 145
146 146 if not walk:
147 147 #pathList.append(path)
148 148 multi_path = path.split(',')
149 149 for single_path in multi_path:
150 150 pathList.append(single_path)
151 151
152 152 else:
153 153 #dirList = []
154 154 multi_path = path.split(',')
155 155 for single_path in multi_path:
156 156 dirList = []
157 157 for thisPath in os.listdir(single_path):
158 158 if not os.path.isdir(os.path.join(single_path,thisPath)):
159 159 continue
160 160 if not isDoyFolder(thisPath):
161 161 continue
162 162
163 163 dirList.append(thisPath)
164 164
165 165 if not(dirList):
166 166 return None, None
167 167
168 168 thisDate = startDate
169 169
170 170 while(thisDate <= endDate):
171 171 year = thisDate.timetuple().tm_year
172 172 doy = thisDate.timetuple().tm_yday
173 173
174 174 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
175 175 if len(matchlist) == 0:
176 176 thisDate += datetime.timedelta(1)
177 177 continue
178 178 for match in matchlist:
179 179 pathList.append(os.path.join(single_path,match))
180 180
181 181 thisDate += datetime.timedelta(1)
182 182
183 183 if pathList == []:
184 184 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
185 185 return None, None
186 186
187 187 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
188 188
189 189 filenameList = []
190 190 datetimeList = []
191 191 pathDict = {}
192 192 filenameList_to_sort = []
193 193
194 194 for i in range(len(pathList)):
195 195
196 196 thisPath = pathList[i]
197 197
198 198 fileList = glob.glob1(thisPath, "*%s" %ext)
199 199 fileList.sort()
200 200 pathDict.setdefault(fileList[0])
201 201 pathDict[fileList[0]] = i
202 202 filenameList_to_sort.append(fileList[0])
203 203
204 204 filenameList_to_sort.sort()
205 205
206 206 for file in filenameList_to_sort:
207 207 thisPath = pathList[pathDict[file]]
208 208
209 209 fileList = glob.glob1(thisPath, "*%s" %ext)
210 210 fileList.sort()
211 211
212 212 for file in fileList:
213 213
214 214 filename = os.path.join(thisPath,file)
215 215 thisDatetime = self.__isFileinThisTime(filename, secStart, secEnd)
216 216
217 217 if not(thisDatetime):
218 218 continue
219 219
220 220 filenameList.append(filename)
221 221 datetimeList.append(thisDatetime)
222 222
223 223 if not(filenameList):
224 224 print "Any file was found for the time range %s - %s" %(startTime, endTime)
225 225 return None, None
226 226
227 227 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
228 228 print
229 229
230 230 for i in range(len(filenameList)):
231 231 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
232 232
233 233 self.filenameList = filenameList
234 234 self.datetimeList = datetimeList
235 235
236 236 return pathList, filenameList
237 237
238 238 def __isFileinThisTime(self, filename, startSeconds, endSeconds):
239 239 """
240 240 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
241 241
242 242 Inputs:
243 243 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
244 244
245 245 startTime : tiempo inicial del rango seleccionado en formato datetime.time
246 246
247 247 endTime : tiempo final del rango seleccionado en formato datetime.time
248 248
249 249 Return:
250 250 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
251 251 fecha especificado, de lo contrario retorna False.
252 252
253 253 Excepciones:
254 254 Si el archivo no existe o no puede ser abierto
255 255 Si la cabecera no puede ser leida.
256 256
257 257 """
258 258
259 259 try:
260 260 fp = fp = h5py.File(filename,'r')
261 261 except IOError:
262 262 traceback.print_exc()
263 263 raise IOError, "The file %s can't be opened" %(filename)
264 264
265 265 grp = fp['Data']
266 266 timeAux = grp['time']
267 267 time0 = timeAux[:][0].astype(numpy.float) #Time Vector
268 268
269 269 fp.close()
270 270
271 271 if self.timezone == 'lt':
272 272 time0 -= 5*3600
273 273
274 274 boolTimer = numpy.logical_and(time0 >= startSeconds,time0 < endSeconds)
275 275
276 276 if not (numpy.any(boolTimer)):
277 277 return None
278 278
279 279 thisDatetime = datetime.datetime.utcfromtimestamp(time0[0])
280 280 return thisDatetime
281 281
282 282 def __checkPath(self):
283 283 if os.path.exists(self.path):
284 284 self.status = 1
285 285 else:
286 286 self.status = 0
287 287 print 'Path:%s does not exists'%self.path
288 288
289 289 return
290 290
291 291 def __setNextFileOffline(self):
292 292 idFile = self.fileIndex
293 293 idFile += 1
294 294
295 295 if not(idFile < len(self.filenameList)):
296 296 print "No more Files"
297 297 return 0
298 298
299 299 filename = self.filenameList[idFile]
300 300
301 301 filePointer = h5py.File(filename,'r')
302 302
303 303 self.flagIsNewFile = 1
304 304 self.fileIndex = idFile
305 305 self.filename = filename
306 306
307 307 self.fp = filePointer
308 308
309 309 print "Setting the file: %s"%self.filename
310 310
311 311 self.__readMetadata()
312 312 self.__setBlockList()
313 313 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
314 314 self.nRecords = self.fp['Data'].attrs['nRecords']
315 315 self.blockIndex = 0
316 316 return 1
317 317
318 318 def __setBlockList(self):
319 319 '''
320 320 self.fp
321 321 self.startDateTime
322 322 self.endDateTime
323 323
324 324 self.blockList
325 325 self.blocksPerFile
326 326
327 327 '''
328 328 filePointer = self.fp
329 329 secStart = self.secStart
330 330 secEnd = self.secEnd
331 331
332 332 grp = filePointer['Data']
333 333 timeVector = grp['time'].value.astype(numpy.float)[0]
334 334
335 335 if self.timezone == 'lt':
336 336 timeVector -= 5*3600
337 337
338 338 ind = numpy.where(numpy.logical_and(timeVector >= secStart , timeVector < secEnd))[0]
339 339
340 340 self.blockList = ind
341 341 self.blocksPerFile = len(ind)
342 342
343 343 return
344 344
345 345 def __readMetadata(self):
346 346 '''
347 347 self.pathMeta
348 348
349 349 self.listShapes
350 350 self.listMetaname
351 351 self.listMeta
352 352
353 353 '''
354 354
355 355 grp = self.fp['Data']
356 356 pathMeta = os.path.join(self.path, grp.attrs['metadata'])
357 357
358 358 if pathMeta == self.pathMeta:
359 359 return
360 360 else:
361 361 self.pathMeta = pathMeta
362 362
363 363 filePointer = h5py.File(self.pathMeta,'r')
364 364 groupPointer = filePointer['Metadata']
365 365
366 366 listMetaname = []
367 367 listMetadata = []
368 368 for item in groupPointer.items():
369 369 name = item[0]
370 370
371 371 if name=='array dimensions':
372 372 table = groupPointer[name][:]
373 373 listShapes = {}
374 374 for shapes in table:
375 375 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4]])
376 376 else:
377 377 data = groupPointer[name].value
378 378 listMetaname.append(name)
379 379 listMetadata.append(data)
380 380
381 381 if name=='type':
382 382 self.__initDataOut(data)
383 383
384 384 filePointer.close()
385 385
386 386 self.listShapes = listShapes
387 387 self.listMetaname = listMetaname
388 388 self.listMeta = listMetadata
389 389
390 390 return
391 391
392 392 def __readData(self):
393 393 grp = self.fp['Data']
394 394 listdataname = []
395 395 listdata = []
396 396
397 397 for item in grp.items():
398 398 name = item[0]
399 399
400 400 if name == 'time':
401 401 listdataname.append('utctime')
402 402 timeAux = grp[name].value.astype(numpy.float)[0]
403 403 listdata.append(timeAux)
404 404 continue
405 405
406 406 listdataname.append(name)
407 407 array = self.__setDataArray(self.nRecords, grp[name],self.listShapes[name])
408 408 listdata.append(array)
409 409
410 410 self.listDataname = listdataname
411 411 self.listData = listdata
412 412 return
413 413
414 414 def __setDataArray(self, nRecords, dataset, shapes):
415 415
416 416 nChannels = shapes[0] #Dimension 0
417 417
418 418 nPoints = shapes[1] #Dimension 1, number of Points or Parameters
419 419
420 420 nSamples = shapes[2] #Dimension 2, number of samples or ranges
421 421
422 422 mode = shapes[3]
423 423
424 424 # if nPoints>1:
425 425 # arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
426 426 # else:
427 427 # arrayData = numpy.zeros((nRecords,nChannels,nSamples))
428 428 #
429 429 # chn = 'channel'
430 430 #
431 431 # for i in range(nChannels):
432 432 #
433 433 # data = dataset[chn + str(i)].value
434 434 #
435 435 # if nPoints>1:
436 436 # data = numpy.rollaxis(data,2)
437 437 #
438 438 # arrayData[:,i,:] = data
439 439
440 440 arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
441 441 doSqueeze = False
442 442 if mode == 0:
443 443 strds = 'channel'
444 444 nDatas = nChannels
445 445 newShapes = (nRecords,nPoints,nSamples)
446 446 if nPoints == 1:
447 447 doSqueeze = True
448 448 axisSqueeze = 2
449 449 else:
450 450 strds = 'param'
451 451 nDatas = nPoints
452 452 newShapes = (nRecords,nChannels,nSamples)
453 453 if nChannels == 1:
454 454 doSqueeze = True
455 455 axisSqueeze = 1
456 456
457 457 for i in range(nDatas):
458 458
459 459 data = dataset[strds + str(i)].value
460 460 data = data.reshape(newShapes)
461 461
462 462 if mode == 0:
463 463 arrayData[:,i,:,:] = data
464 464 else:
465 465 arrayData[:,:,i,:] = data
466 466
467 467 if doSqueeze:
468 468 arrayData = numpy.squeeze(arrayData, axis=axisSqueeze)
469 469
470 470 return arrayData
471 471
472 472 def __initDataOut(self, type):
473 473
474 474 # if type =='Parameters':
475 475 # self.dataOut = Parameters()
476 476 # elif type =='Spectra':
477 477 # self.dataOut = Spectra()
478 478 # elif type =='Voltage':
479 479 # self.dataOut = Voltage()
480 480 # elif type =='Correlation':
481 481 # self.dataOut = Correlation()
482 482
483 483 return
484 484
485 485 def __setDataOut(self):
486 486 listMeta = self.listMeta
487 487 listMetaname = self.listMetaname
488 488 listDataname = self.listDataname
489 489 listData = self.listData
490 490
491 491 blockIndex = self.blockIndex
492 492 blockList = self.blockList
493 493
494 494 for i in range(len(listMeta)):
495 495 setattr(self.dataOut,listMetaname[i],listMeta[i])
496 496
497 497 for j in range(len(listData)):
498 498 if listDataname[j]=='utctime':
499 499 # setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex]])
500 500 setattr(self.dataOut,'utctimeInit',listData[j][blockList[blockIndex]])
501 501 continue
502 502
503 503 setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex],:])
504 504
505 505 return self.dataOut.data_param
506 506
507 507 def getData(self):
508 508
509 509 # if self.flagNoMoreFiles:
510 510 # self.dataOut.flagNoData = True
511 511 # print 'Process finished'
512 512 # return 0
513 513 #
514 514 if self.blockIndex==self.blocksPerFile:
515 515 if not( self.__setNextFileOffline() ):
516 516 self.dataOut.flagNoData = True
517 517 return 0
518 518
519 519 #
520 520 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
521 521 # self.dataOut.flagNoData = True
522 522 # return 0
523 523
524 524 self.__readData()
525 525 self.__setDataOut()
526 526 self.dataOut.flagNoData = False
527 527
528 528 self.blockIndex += 1
529 529
530 530 return
531 531
532 532 def run(self, **kwargs):
533 533
534 534 if not(self.isConfig):
535 535 self.setup(**kwargs)
536 536 # self.setObjProperties()
537 537 self.isConfig = True
538 538
539 539 self.getData()
540 540
541 541 return
542 542
543 543 class HDF5Writer(Operation):
544 544
545 545 ext = ".hdf5"
546 546
547 547 optchar = "D"
548 548
549 549 metaoptchar = "M"
550 550
551 551 metaFile = None
552 552
553 553 filename = None
554 554
555 555 path = None
556 556
557 557 setFile = None
558 558
559 559 fp = None
560 560
561 561 grp = None
562 562
563 563 ds = None
564 564
565 565 firsttime = True
566 566
567 567 #Configurations
568 568
569 569 blocksPerFile = None
570 570
571 571 blockIndex = None
572 572
573 573 dataOut = None
574 574
575 575 #Data Arrays
576 576
577 577 dataList = None
578 578
579 579 metadataList = None
580 580
581 581 arrayDim = None
582 582
583 583 tableDim = None
584 584
585 585 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
586 586
587 587 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
588 588
589 589 mode = None
590 590
591 591 nDatas = None #Number of datasets to be stored per array
592 592
593 593 nDims = None #Number Dimensions in each dataset
594 594
595 595 nDimsForDs = None
596 596
597 597 def __init__(self):
598 598
599 599 Operation.__init__(self)
600 600 self.isConfig = False
601 601 return
602 602
603 603
604 604 def setup(self, dataOut, **kwargs):
605 605
606 606 self.path = kwargs['path']
607 607
608 608 if kwargs.has_key('ext'):
609 609 self.ext = kwargs['ext']
610 610
611 611 if kwargs.has_key('blocksPerFile'):
612 612 self.blocksPerFile = kwargs['blocksPerFile']
613 613 else:
614 614 self.blocksPerFile = 10
615 615
616 616 self.metadataList = kwargs['metadataList']
617 617
618 618 self.dataList = kwargs['dataList']
619 619
620 620 self.dataOut = dataOut
621 621
622 622 if kwargs.has_key('mode'):
623 623 mode = kwargs['mode']
624 624
625 625 if type(mode) == int:
626 626 mode = numpy.zeros(len(self.dataList)) + mode
627 627 else:
628 628 mode = numpy.zeros(len(self.dataList))
629 629
630 630 self.mode = mode
631 631
632 632 arrayDim = numpy.zeros((len(self.dataList),5))
633 633
634 634 #Table dimensions
635 635
636 636 dtype0 = self.dtype
637 637
638 638 tableList = []
639 639
640 640 for i in range(len(self.dataList)):
641 641
642 642 dataAux = getattr(self.dataOut, self.dataList[i])
643 643
644 644 if type(dataAux)==float or type(dataAux)==int:
645 645 arrayDim[i,0] = 1
646 646 else:
647 647 arrayDim0 = dataAux.shape
648 648 arrayDim[i,0] = len(arrayDim0)
649 649 arrayDim[i,4] = mode[i]
650 650
651 651 if len(arrayDim0) == 3:
652 652 arrayDim[i,1:-1] = numpy.array(arrayDim0)
653 653 elif len(arrayDim0) == 2:
654 654 arrayDim[i,2:-1] = numpy.array(arrayDim0) #nHeights
655 655 elif len(arrayDim0) == 1:
656 656 arrayDim[i,3] = arrayDim0
657 657 elif len(arrayDim0) == 0:
658 658 arrayDim[i,0] = 1
659 659 arrayDim[i,3] = 1
660 660
661 661 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
662 662 tableList.append(table)
663 663
664 664 self.arrayDim = arrayDim
665 665 self.tableDim = numpy.array(tableList, dtype = dtype0)
666 666 self.blockIndex = 0
667 667
668 668 return
669 669
670 670 def putMetadata(self):
671 671
672 672 fp = self.createMetadataFile()
673 673 self.writeMetadata(fp)
674 674 fp.close()
675 675 return
676 676
677 677 def createMetadataFile(self):
678 678 ext = self.ext
679 679 path = self.path
680 680 setFile = self.setFile
681 681
682 682 timeTuple = time.localtime(self.dataOut.utctime)
683 683
684 684 subfolder = ''
685 685 fullpath = os.path.join( path, subfolder )
686 686
687 687 if not( os.path.exists(fullpath) ):
688 688 os.mkdir(fullpath)
689 689 setFile = -1 #inicializo mi contador de seteo
690 690
691 691 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
692 692 fullpath = os.path.join( path, subfolder )
693 693
694 694 if not( os.path.exists(fullpath) ):
695 695 os.mkdir(fullpath)
696 696 setFile = -1 #inicializo mi contador de seteo
697 697
698 698 else:
699 699 filesList = os.listdir( fullpath )
700 700 filesList = sorted( filesList, key=str.lower )
701 701 if len( filesList ) > 0:
702 702 filesList = [k for k in filesList if 'M' in k]
703 703 filen = filesList[-1]
704 704 # el filename debera tener el siguiente formato
705 705 # 0 1234 567 89A BCDE (hex)
706 706 # x YYYY DDD SSS .ext
707 707 if isNumber( filen[8:11] ):
708 708 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
709 709 else:
710 710 setFile = -1
711 711 else:
712 712 setFile = -1 #inicializo mi contador de seteo
713 713
714 714 setFile += 1
715 715
716 716 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
717 717 timeTuple.tm_year,
718 718 timeTuple.tm_yday,
719 719 setFile,
720 720 ext )
721 721
722 722 filename = os.path.join( path, subfolder, file )
723 723 self.metaFile = file
724 724 #Setting HDF5 File
725 725 fp = h5py.File(filename,'w')
726 726
727 727 return fp
728 728
729 729 def writeMetadata(self, fp):
730 730
731 731 grp = fp.create_group("Metadata")
732 732 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
733 733
734 734 for i in range(len(self.metadataList)):
735 735 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
736 736 return
737 737
738 738 def setNextFile(self):
739 739
740 740 ext = self.ext
741 741 path = self.path
742 742 setFile = self.setFile
743 743 mode = self.mode
744 744
745 745 timeTuple = time.localtime(self.dataOut.utctime)
746 746 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
747 747
748 748 fullpath = os.path.join( path, subfolder )
749 749
750 750 if os.path.exists(fullpath):
751 751 filesList = os.listdir( fullpath )
752 752 filesList = [k for k in filesList if 'D' in k]
753 753 if len( filesList ) > 0:
754 754 filesList = sorted( filesList, key=str.lower )
755 755 filen = filesList[-1]
756 756 # el filename debera tener el siguiente formato
757 757 # 0 1234 567 89A BCDE (hex)
758 758 # x YYYY DDD SSS .ext
759 759 if isNumber( filen[8:11] ):
760 760 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
761 761 else:
762 762 setFile = -1
763 763 else:
764 764 setFile = -1 #inicializo mi contador de seteo
765 765
766 766 setFile += 1
767 767
768 768 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
769 769 timeTuple.tm_year,
770 770 timeTuple.tm_yday,
771 771 setFile,
772 772 ext )
773 773
774 774 filename = os.path.join( path, subfolder, file )
775 775
776 776 #Setting HDF5 File
777 777 fp = h5py.File(filename,'w')
778 778 grp = fp.create_group("Data")
779 779 grp.attrs['metadata'] = self.metaFile
780 780
781 781 # grp.attrs['blocksPerFile'] = 0
782 782
783 783 ds = []
784 784 data = []
785 785 nDimsForDs = []
786 786
787 787 nDatas = numpy.zeros(len(self.dataList))
788 788 nDims = self.arrayDim[:,0]
789 789
790 790 nDim1 = self.arrayDim[:,2]
791 791 nDim0 = self.arrayDim[:,3]
792 792
793 793 for i in range(len(self.dataList)):
794 794
795 795 if nDims[i]==1:
796 796 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
797 797 ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
798 798 ds.append(ds0)
799 799 data.append([])
800 800 nDimsForDs.append(nDims[i])
801 801 else:
802 802
803 803 if mode[i]==0:
804 804 strMode = "channel"
805 805 nDatas[i] = self.arrayDim[i,1]
806 806 else:
807 807 strMode = "param"
808 808 nDatas[i] = self.arrayDim[i,2]
809 809
810 810 if nDims[i]==2:
811 811 nDatas[i] = self.arrayDim[i,2]
812 812
813 813 grp0 = grp.create_group(self.dataList[i])
814 814
815 815 for j in range(int(nDatas[i])):
816 816 tableName = strMode + str(j)
817 817
818 818 if nDims[i] == 3:
819 819 ds0 = grp0.create_dataset(tableName, (nDim1[i],nDim0[i],1) , data = numpy.zeros((nDim1[i],nDim0[i],1)) ,maxshape=(None,nDim0[i],None), chunks=True)
820 820 else:
821 821 ds0 = grp0.create_dataset(tableName, (1,nDim0[i]), data = numpy.zeros((1,nDim0[i])) , maxshape=(None,nDim0[i]), chunks=True)
822 822
823 823 ds.append(ds0)
824 824 data.append([])
825 825 nDimsForDs.append(nDims[i])
826 826 self.nDatas = nDatas
827 827 self.nDims = nDims
828 828 self.nDimsForDs = nDimsForDs
829 829 #Saving variables
830 830 print 'Writing the file: %s'%filename
831 831 self.filename = filename
832 832 self.fp = fp
833 833 self.grp = grp
834 834 self.grp.attrs.modify('nRecords', 1)
835 835 self.ds = ds
836 836 self.data = data
837 837
838 838 self.setFile = setFile
839 839 self.firsttime = True
840 840 self.blockIndex = 0
841 841 return
842 842
843 843 def putData(self):
844 844
845 845 if not self.firsttime:
846 self.fp.flush()
847 self.fp.close()
848 846 self.readBlock()
849 847
850 848 if self.blockIndex == self.blocksPerFile:
851 849
852 850 self.setNextFile()
853 851
854 852 self.setBlock()
855 853 self.writeBlock()
856 854
855 self.fp.flush()
856 self.fp.close()
857 857
858 858 return
859 859
860 860 def readBlock(self):
861 861
862 862 '''
863 863 data Array configured
864 864
865 865
866 866 self.data
867 867 '''
868 868 ds = self.ds
869 869 #Setting HDF5 File
870 870 fp = h5py.File(self.filename,'r+')
871 871 grp = fp["Data"]
872 872 ind = 0
873 873
874 874 # grp.attrs['blocksPerFile'] = 0
875 875 for i in range(len(self.dataList)):
876 876
877 877 if self.nDims[i]==1:
878 878 ds0 = grp[self.dataList[i]]
879 879 ds[ind] = ds0
880 880 ind += 1
881 881 else:
882 882 if self.mode[i]==0:
883 883 strMode = "channel"
884 884 else:
885 885 strMode = "param"
886 886
887 887 grp0 = grp[self.dataList[i]]
888 888
889 889 for j in range(int(self.nDatas[i])):
890 890 tableName = strMode + str(j)
891 891 ds0 = grp0[tableName]
892 892 ds[ind] = ds0
893 893 ind += 1
894 894
895 895
896 896 self.fp = fp
897 897 self.grp = grp
898 898 self.ds = ds
899 899
900 900 return
901 901
902 902
903 903 def setBlock(self):
904 904 '''
905 905 data Array configured
906 906
907 907
908 908 self.data
909 909 '''
910 910 #Creating Arrays
911 911 data = self.data
912 912 nDatas = self.nDatas
913 913 nDims = self.nDims
914 914 mode = self.mode
915 915 ind = 0
916 916
917 917 for i in range(len(self.dataList)):
918 918 dataAux = getattr(self.dataOut,self.dataList[i])
919 919
920 920 if nDims[i] == 1:
921 921 # data[ind] = numpy.array([str(dataAux)]).reshape((1,1))
922 922 data[ind] = dataAux
923 923 # if not self.firsttime:
924 924 # data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind]))
925 925 ind += 1
926 926 else:
927 927 for j in range(int(nDatas[i])):
928 928 if (mode[i] == 0) or (nDims[i] == 2): #In case division per channel or Dimensions is only 1
929 929 data[ind] = dataAux[j,:]
930 930 else:
931 931 data[ind] = dataAux[:,j,:]
932 932
933 933 # if nDims[i] == 3:
934 934 # data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1))
935 935
936 936 # if not self.firsttime:
937 937 # data[ind] = numpy.dstack((self.ds[ind][:], data[ind]))
938 938
939 939 # else:
940 940 # data[ind] = data[ind].reshape((1,data[ind].shape[0]))
941 941
942 942 # if not self.firsttime:
943 943 # data[ind] = numpy.vstack((self.ds[ind][:], data[ind]))
944 944 ind += 1
945 945
946 946 self.data = data
947 947 return
948 948
949 949 def writeBlock(self):
950 950 '''
951 951 Saves the block in the HDF5 file
952 952 '''
953 953 for i in range(len(self.ds)):
954 954 if self.firsttime:
955 955 # self.ds[i].resize(self.data[i].shape)
956 956 # self.ds[i][self.blockIndex,:] = self.data[i]
957 957 if type(self.data[i]) == numpy.ndarray:
958 958 nDims1 = len(self.ds[i].shape)
959 959
960 960 if nDims1 == 3:
961 961 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
962 962
963 963 self.ds[i].resize(self.data[i].shape)
964 964 self.ds[i][:] = self.data[i]
965 965 else:
966 966 if self.nDimsForDs[i] == 1:
967 967 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
968 968 self.ds[i][0,-1] = self.data[i]
969 969 elif self.nDimsForDs[i] == 2:
970 970 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
971 971 self.ds[i][self.blockIndex,:] = self.data[i]
972 972 elif self.nDimsForDs[i] == 3:
973 973
974 974 dataShape = self.data[i].shape
975 975 dsShape = self.ds[i].shape
976 976
977 977 if dataShape[0]==dsShape[0]:
978 978 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
979 979 self.ds[i][:,:,-1] = self.data[i]
980 980 else:
981 981 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1],self.ds[i].shape[2]))
982 982 self.ds[i][dsShape[0]:,:,0] = self.data[i]
983 983 # self.ds[i].append(self.data[i])
984 984 # self.fp.flush()
985 985 # if not self.firsttime:
986 986 # self.fp.root.Data._v_attrs.nRecords = self.blockIndex
987 987
988 988 # if self.firsttime:
989 989 # self.fp.close()
990 990 # self.readBlock2()
991 991
992 992 self.blockIndex += 1
993 993 self.firsttime = False
994 994 return
995 995
996 996 def run(self, dataOut, **kwargs):
997 997 if not(self.isConfig):
998 998 self.setup(dataOut, **kwargs)
999 999 self.isConfig = True
1000 1000 self.putMetadata()
1001 1001 self.setNextFile()
1002 1002
1003 1003 self.putData()
1004 1004 return
1005 1005
General Comments 0
You need to be logged in to leave comments. Login now