##// END OF EJS Templates
Version 2.2.5 Fixed several bugs, add jro colormap for spectra/rti, add ParamWriter, TODO: Fix decimation currently disabled
Juan C. Valdez -
r860:5c2852893069
parent child
Show More
@@ -1,29 +1,12
1 Prerequisites:
2
3 Core:
4 -numpy 1.8.0
5 -scipy
6 -math
7 -matplotlib
8 -h5py
9 -ftplib
10 -paramiko (optional for SendTFilesToServer)
11 -stuffr (optional for jroIO_hf)
12 -pyfits (Fits data)
13
14 GUI:
15 -PyQt4
16 -wxPython
17
18 1 Signal Chain Installation:
19 2
20 1. Install numpy, matplotlib, TKAgg
3 1. Install system dependencies: python-pip python-dev gfortran libpng-dev freetype* libblas-dev liblapack-dev libatlas-base-dev python-qt4
21 4 2. Install digital_rf_hdf5 module (developed by Haystack Observatory)
22 5 if you want to use USRP data
23 6 3. untar schainpy-x.x.x.tar.gz
24 7 4. cd schainpy-x.x.x
25 8 5. execute:
26 [hostname]$ sudo pyhon setup.py install
9 [hostname]$ sudo pip install ./
27 10 6. testing gui:
28 11 [hostname]$ schainGUI (enter)
29 12
@@ -343,7 +343,7 class OperationConf():
343 343
344 344 self.parmConfObjList = []
345 345
346 parmElementList = opElement.getiterator(ParameterConf().getElementName())
346 parmElementList = opElement.iter(ParameterConf().getElementName())
347 347
348 348 for parmElement in parmElementList:
349 349 parmConfObj = ParameterConf()
@@ -568,7 +568,7 class ProcUnitConf():
568 568
569 569 self.opConfObjList = []
570 570
571 opElementList = upElement.getiterator(OperationConf().getElementName())
571 opElementList = upElement.iter(OperationConf().getElementName())
572 572
573 573 for opElement in opElementList:
574 574 opConfObj = OperationConf()
@@ -798,7 +798,7 class ReadUnitConf(ProcUnitConf):
798 798
799 799 self.opConfObjList = []
800 800
801 opElementList = upElement.getiterator(OperationConf().getElementName())
801 opElementList = upElement.iter(OperationConf().getElementName())
802 802
803 803 for opElement in opElementList:
804 804 opConfObj = OperationConf()
@@ -1026,7 +1026,7 class Project():
1026 1026 self.name = self.projectElement.get('name')
1027 1027 self.description = self.projectElement.get('description')
1028 1028
1029 readUnitElementList = self.projectElement.getiterator(ReadUnitConf().getElementName())
1029 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1030 1030
1031 1031 for readUnitElement in readUnitElementList:
1032 1032 readUnitConfObj = ReadUnitConf()
@@ -1037,7 +1037,7 class Project():
1037 1037
1038 1038 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1039 1039
1040 procUnitElementList = self.projectElement.getiterator(ProcUnitConf().getElementName())
1040 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1041 1041
1042 1042 for procUnitElement in procUnitElementList:
1043 1043 procUnitConfObj = ProcUnitConf()
@@ -312,7 +312,7 class Axes:
312 312 decimationy = None
313 313
314 314 __MAXNUMX = 200
315 __MAXNUMY = 400
315 __MAXNUMY = 100
316 316
317 317 __MAXNUMTIME = 500
318 318
@@ -501,13 +501,15 class Axes:
501 501 xlen = len(x)
502 502 ylen = len(y)
503 503
504 decimationx = numpy.floor(xlen/self.__MAXNUMX) - 1 if numpy.floor(xlen/self.__MAXNUMX)>1 else 1
505 decimationy = numpy.floor(ylen/self.__MAXNUMY) + 1
504 decimationx = int(xlen/self.__MAXNUMX)+1 \
505 if int(xlen/self.__MAXNUMX)>1 else 1
506 decimationy = int(ylen/self.__MAXNUMY) \
507 if int(ylen/self.__MAXNUMY)>1 else 1
506 508
509 x_buffer = x#[::decimationx]
510 y_buffer = y#[::decimationy]
511 z_buffer = z#[::decimationx, ::decimationy]
507 512
508 x_buffer = x[::decimationx]
509 y_buffer = y[::decimationy]
510 z_buffer = z[::decimationx, ::decimationy]
511 513 #===================================================
512 514
513 515 if self.__firsttime:
@@ -604,12 +606,14 class Axes:
604 606 xlen = len(self.x_buffer)
605 607 ylen = len(y)
606 608
607 decimationx = numpy.floor(xlen/maxNumX) + 1
608 decimationy = numpy.floor(ylen/maxNumY) + 1
609 decimationx = int(xlen/self.__MAXNUMX) \
610 if int(xlen/self.__MAXNUMX)>1 else 1
611 decimationy = int(ylen/self.__MAXNUMY) \
612 if int(ylen/self.__MAXNUMY)>1 else 1
609 613
610 x_buffer = self.x_buffer[::decimationx]
611 y_buffer = y[::decimationy]
612 z_buffer = z_buffer[::decimationx, ::decimationy]
614 x_buffer = self.x_buffer#[::decimationx]
615 y_buffer = y#[::decimationy]
616 z_buffer = z_buffer#[::decimationx, ::decimationy]
613 617 #===================================================
614 618
615 619 x_buffer, y_buffer, z_buffer = self.__fillGaps(x_buffer, y_buffer, z_buffer)
@@ -5,6 +5,8 import numpy
5 5 from figure import Figure, isRealtime, isTimeInHourRange
6 6 from plotting_codes import *
7 7
8 import matplotlib.pyplot as plt
9
8 10 class MomentsPlot(Figure):
9 11
10 12 isConfig = None
@@ -446,11 +448,10 class WindProfilerPlot(Figure):
446 448 # tmax = None
447 449
448 450 x = dataOut.getTimeRange1(dataOut.outputInterval)
449 # y = dataOut.heightList
450 451 y = dataOut.heightList
451
452 452 z = dataOut.data_output.copy()
453 453 nplots = z.shape[0] #Number of wind dimensions estimated
454
454 455 nplotsw = nplots
455 456
456 457 #If there is a SNR function defined
@@ -471,7 +472,8 class WindProfilerPlot(Figure):
471 472
472 473 # showprofile = False
473 474 # thisDatetime = dataOut.datatime
474 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
475 #thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
476 thisDatetime = datetime.datetime.now()
475 477 title = wintitle + "Wind"
476 478 xlabel = ""
477 479 ylabel = "Height (km)"
@@ -490,8 +492,8 class WindProfilerPlot(Figure):
490 492
491 493 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
492 494
493 if ymin == None: ymin = numpy.nanmin(y)
494 if ymax == None: ymax = numpy.nanmax(y)
495 #if ymin == None: ymin = numpy.nanmin(y)
496 #if ymax == None: ymax = numpy.nanmax(y)
495 497
496 498 if zmax == None: zmax = numpy.nanmax(abs(z[range(2),:]))
497 499 #if numpy.isnan(zmax): zmax = 50
@@ -501,9 +503,9 class WindProfilerPlot(Figure):
501 503 if zmax_ver == None: zmax_ver = numpy.nanmax(abs(z[2,:]))
502 504 if zmin_ver == None: zmin_ver = -zmax_ver
503 505
504 if dataOut.data_SNR is not None:
505 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
506 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
506 # if dataOut.data_SNR is not None:
507 # if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
508 # if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
507 509
508 510
509 511 self.FTP_WEI = ftp_wei
@@ -518,8 +520,8 class WindProfilerPlot(Figure):
518 520
519 521 self.setWinTitle(title)
520 522
521 if ((self.xmax - x[1]) < (x[1]-x[0])):
522 x[1] = self.xmax
523 #if ((self.xmax - x[1]) < (x[1]-x[0])):
524 # x[1] = self.xmax
523 525
524 526 strWind = ['Zonal', 'Meridional', 'Vertical']
525 527 strCb = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)']
@@ -561,7 +563,7 class WindProfilerPlot(Figure):
561 563 thisDatetime=thisDatetime,
562 564 update_figfile=update_figfile)
563 565
564 if dataOut.ltctime + dataOut.outputInterval >= self.xmax:
566 if False and dataOut.ltctime + dataOut.outputInterval >= self.xmax:
565 567 self.counter_imagwr = wr_period
566 568 self.isConfig = False
567 569 update_figfile = True
@@ -778,7 +780,7 class ParametersPlot(Figure):
778 780
779 781
780 782
781 class Parameters1Plot(Figure):
783 class ParametersPlot(Figure):
782 784
783 785 __isConfig = None
784 786 __nsubplots = None
@@ -86,7 +86,7 class SpectraPlot(Figure):
86 86 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
87 87 server=None, folder=None, username=None, password=None,
88 88 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
89 xaxis="frequency", colormap='jet'):
89 xaxis="velocity", **kwargs):
90 90
91 91 """
92 92
@@ -104,6 +104,8 class SpectraPlot(Figure):
104 104 zmax : None
105 105 """
106 106
107 colormap = kwargs.get('colormap','jet')
108
107 109 if realtime:
108 110 if not(isRealtime(utcdatatime = dataOut.utctime)):
109 111 print 'Skipping this plot function'
@@ -514,10 +516,10 class RTIPlot(Figure):
514 516
515 517 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
516 518 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
517 timerange=None, colormap='jet',
519 timerange=None,
518 520 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
519 521 server=None, folder=None, username=None, password=None,
520 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
522 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, **kwargs):
521 523
522 524 """
523 525
@@ -535,6 +537,7 class RTIPlot(Figure):
535 537 zmax : None
536 538 """
537 539
540 colormap = kwargs.get('colormap', 'jet')
538 541 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
539 542 return
540 543
@@ -18,6 +18,8 from matplotlib.ticker import FuncFormatter, LinearLocator
18 18 #Actualizacion de las funciones del driver
19 19 ###########################################
20 20
21 # create jro colormap
22
21 23 jet_values = matplotlib.pyplot.get_cmap("jet", 100)(numpy.arange(100))[10:90]
22 24 blu_values = matplotlib.pyplot.get_cmap("seismic_r", 20)(numpy.arange(20))[10:15]
23 25 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list("jro", numpy.vstack((blu_values, jet_values)))
1 NO CONTENT: modified file
@@ -399,70 +399,7 class ParamReader(ProcessingUnit):
399 399 self.listData = listdata
400 400 return
401 401
402 def __setDataArray(self, dataset, shapes):
403 402
404 nDims = shapes[0]
405
406 nDim2 = shapes[1] #Dimension 0
407
408 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
409
410 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
411
412 mode = shapes[4] #Mode of storing
413
414 blockList = self.blockList
415
416 blocksPerFile = self.blocksPerFile
417
418 #Depending on what mode the data was stored
419 if mode == 0: #Divided in channels
420 arrayData = dataset.value.astype(numpy.float)[0][blockList]
421 if mode == 1: #Divided in parameter
422 strds = 'table'
423 nDatas = nDim1
424 newShapes = (blocksPerFile,nDim2,nDim0)
425 elif mode==2: #Concatenated in a table
426 strds = 'table0'
427 arrayData = dataset[strds].value
428 #Selecting part of the dataset
429 utctime = arrayData[:,0]
430 u, indices = numpy.unique(utctime, return_index=True)
431
432 if blockList.size != indices.size:
433 indMin = indices[blockList[0]]
434 if blockList[-1] + 1 >= indices.size:
435 arrayData = arrayData[indMin:,:]
436 else:
437 indMax = indices[blockList[-1] + 1]
438 arrayData = arrayData[indMin:indMax,:]
439 return arrayData
440
441 #------- One dimension ---------------
442 if nDims == 0:
443 arrayData = dataset.value.astype(numpy.float)[0][blockList]
444
445 #------- Two dimensions -----------
446 elif nDims == 2:
447 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
448 newShapes = (blocksPerFile,nDim0)
449 nDatas = nDim1
450
451 for i in range(nDatas):
452 data = dataset[strds + str(i)].value
453 arrayData[:,i,:] = data[blockList,:]
454
455 #------- Three dimensions ---------
456 else:
457 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
458 for i in range(nDatas):
459
460 data = dataset[strds + str(i)].value
461
462 for b in range(blockList.size):
463 arrayData[b,:,i,:] = data[:,:,blockList[b]]
464
465 return arrayData
466 403
467 404 def __setDataOut(self):
468 405 listMeta = self.listMeta
@@ -571,7 +571,9 class SpectraWriter(JRODataWriter, Operation):
571 571
572 572 if self.dataOut.flagDiscontinuousBlock:
573 573 self.data_spc.fill(0)
574 if self.dataOut.data_cspc is not None:
574 575 self.data_cspc.fill(0)
576 if self.dataOut.data_dc is not None:
575 577 self.data_dc.fill(0)
576 578 self.setNextFile()
577 579
1 NO CONTENT: modified file
@@ -33,6 +33,8 class SpectraHeisProc(ProcessingUnit):
33 33 self.dataOut.nCode = self.dataIn.nCode
34 34 self.dataOut.code = self.dataIn.code
35 35 # self.dataOut.nProfiles = 1
36 self.dataOut.ippFactor = 1
37 self.dataOut.noise_estimation = None
36 38 # self.dataOut.nProfiles = self.dataOut.nFFTPoints
37 39 self.dataOut.nFFTPoints = self.dataIn.nHeights
38 40 # self.dataOut.channelIndexList = self.dataIn.channelIndexList
@@ -5,4 +5,4 $Id: Processor.py 1 2012-11-12 18:56:07Z murco $
5 5 '''
6 6
7 7 from jroutils_ftp import *
8 from jroutils_publish import PublishData No newline at end of file
8 from jroutils_publish import *
@@ -9,11 +9,11 import paho.mqtt.client as mqtt
9 9
10 10 from schainpy.model.proc.jroproc_base import Operation
11 11
12
13 12 class PrettyFloat(float):
14 13 def __repr__(self):
15 14 return '%.2f' % self
16 15
16
17 17 def pretty_floats(obj):
18 18 if isinstance(obj, float):
19 19 return PrettyFloat(obj)
@@ -23,80 +23,130 def pretty_floats(obj):
23 23 return map(pretty_floats, obj)
24 24 return obj
25 25
26
26 27 class PublishData(Operation):
28 """Clase publish."""
27 29
28 30 __MAXNUMX = 80
29 31 __MAXNUMY = 80
30 32
31 33 def __init__(self):
32
34 """Inicio."""
33 35 Operation.__init__(self)
34
35 36 self.isConfig = False
36 37 self.client = None
37 38
38 @staticmethod
39 def on_disconnect(client, userdata, rc):
39 def on_disconnect(self, client, userdata, rc):
40 40 if rc != 0:
41 41 print("Unexpected disconnection.")
42 self.connect()
42 43
43 def setup(self, host, port=1883, username=None, password=None, **kwargs):
44
45 self.client = mqtt.Client()
44 def connect(self):
45 print 'trying to connect'
46 46 try:
47 self.client.connect(host=host, port=port, keepalive=60*10, bind_address='')
47 self.client.connect(
48 host=self.host,
49 port=self.port,
50 keepalive=60*10,
51 bind_address='')
52 print "connected"
53 self.client.loop_start()
54 # self.client.publish(
55 # self.topic + 'SETUP',
56 # json.dumps(setup),
57 # retain=True
58 # )
48 59 except:
60 print "MQTT Conection error."
49 61 self.client = False
62
63 def setup(self, host, port=1883, username=None, password=None, **kwargs):
64
50 65 self.topic = kwargs.get('topic', 'schain')
51 66 self.delay = kwargs.get('delay', 0)
67 self.plottype = kwargs.get('plottype', 'spectra')
52 68 self.host = host
53 69 self.port = port
54 70 self.cnt = 0
55
56 def run(self, dataOut, host, datatype='data_spc', **kwargs):
57
58 if not self.isConfig:
59 self.setup(host, **kwargs)
60 self.isConfig = True
61
62 data = getattr(dataOut, datatype)
63
64 z = data/dataOut.normFactor
71 setup = []
72 for plot in self.plottype:
73 setup.append({
74 'plot': plot,
75 'topic': self.topic + plot,
76 'title': getattr(self, plot + '_' + 'title', False),
77 'xlabel': getattr(self, plot + '_' + 'xlabel', False),
78 'ylabel': getattr(self, plot + '_' + 'ylabel', False),
79 'xrange': getattr(self, plot + '_' + 'xrange', False),
80 'yrange': getattr(self, plot + '_' + 'yrange', False),
81 'zrange': getattr(self, plot + '_' + 'zrange', False),
82 })
83 self.client = mqtt.Client(
84 client_id='jc'+self.topic + 'SCHAIN',
85 clean_session=True)
86 self.client.on_disconnect = self.on_disconnect
87 self.connect()
88
89 def publish_data(self, plottype):
90 data = getattr(self.dataOut, 'data_spc')
91 if plottype == 'spectra':
92 z = data/self.dataOut.normFactor
65 93 zdB = 10*numpy.log10(z)
66 avg = numpy.average(z, axis=1)
67 avgdB = 10*numpy.log10(avg)
68
69 94 xlen ,ylen = zdB[0].shape
70
71
72 95 dx = numpy.floor(xlen/self.__MAXNUMX) + 1
73 96 dy = numpy.floor(ylen/self.__MAXNUMY) + 1
74
75 Z = [0 for i in dataOut.channelList]
76 AVG = [0 for i in dataOut.channelList]
77
78 for i in dataOut.channelList:
97 Z = [0 for i in self.dataOut.channelList]
98 for i in self.dataOut.channelList:
79 99 Z[i] = zdB[i][::dx, ::dy].tolist()
80 AVG[i] = avgdB[i][::dy].tolist()
81
82 payload = {'timestamp':dataOut.utctime,
100 payload = {
101 'timestamp': self.dataOut.utctime,
83 102 'data':pretty_floats(Z),
84 'data_profile':pretty_floats(AVG),
85 'channels': ['Ch %s' % ch for ch in dataOut.channelList],
86 'interval': dataOut.getTimeInterval(),
103 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
104 'interval': self.dataOut.getTimeInterval(),
105 'xRange': [0, 80]
87 106 }
88 107
108 elif plottype in ('rti', 'power'):
109 z = data/self.dataOut.normFactor
110 avg = numpy.average(z, axis=1)
111 avgdB = 10*numpy.log10(avg)
112 xlen, ylen = z[0].shape
113 dy = numpy.floor(ylen/self.__MAXNUMY) + 1
114 AVG = [0 for i in self.dataOut.channelList]
115 for i in self.dataOut.channelList:
116 AVG[i] = avgdB[i][::dy].tolist()
117 payload = {
118 'timestamp': self.dataOut.utctime,
119 'data': pretty_floats(AVG),
120 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
121 'interval': self.dataOut.getTimeInterval(),
122 'xRange': [0, 80]
123 }
124 elif plottype == 'noise':
125 noise = self.dataOut.getNoise()/self.dataOut.normFactor
126 noisedB = 10*numpy.log10(noise)
127 payload = {
128 'timestamp': self.dataOut.utctime,
129 'data': pretty_floats(noisedB.reshape(-1, 1).tolist()),
130 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
131 'interval': self.dataOut.getTimeInterval(),
132 'xRange': [0, 80]
133 }
89 134
90 #if self.cnt==self.interval and self.client:
91 135 print 'Publishing data to {}'.format(self.host)
92 self.client.publish(self.topic, json.dumps(payload), qos=0)
93 time.sleep(self.delay)
94 #self.cnt = 0
95 #else:
96 # self.cnt += 1
136 print '*************************'
137 self.client.publish(self.topic + plottype, json.dumps(payload), qos=0)
97 138
98 139
99 def close(self):
140 def run(self, dataOut, host, **kwargs):
141 self.dataOut = dataOut
142 if not self.isConfig:
143 self.setup(host, **kwargs)
144 self.isConfig = True
100 145
146 map(self.publish_data, self.plottype)
147 time.sleep(self.delay)
148
149 def close(self):
101 150 if self.client:
151 self.client.loop_stop()
102 152 self.client.disconnect()
@@ -33,8 +33,12 setup(name="schainpy",
33 33 include_package_data=False,
34 34 scripts =['schainpy/gui/schainGUI',
35 35 'schainpy/scripts/schain'],
36 install_requires=["numpy >= 1.6.0",
36 install_requires=[
37 37 "scipy >= 0.9.0",
38 "h5py >= 2.0.1",
38 39 "matplotlib >= 1.0.0",
40 "pyfits >= 2.0.0",
41 "numpy >= 1.6.0",
42 "paramiko",
39 43 ],
40 44 ) No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now