schain Commit - r860:5c2852893069 · Jicamarca Repository

Version 2.2.5 Fixed several bugs, add jro colormap for spectra/rti, add ParamWriter, TODO: Fix decimation currently disabled

Juan C. Valdez -

r860:5c2852893069

parent child

Context file:

r860:5c2852893069

Collapse all files

README +2 -19

@@ -1,29 +1,12
1	Prerequisites:
2
3	Core:
4	-numpy 1.8.0
5	-scipy
6	-math
7	-matplotlib
8	-h5py
9	-ftplib
10	-paramiko (optional for SendTFilesToServer)
11	-stuffr (optional for jroIO_hf)
12	-pyfits (Fits data)
13
14	GUI:
15	-PyQt4
16	-wxPython
17
18	Signal Chain Installation:	1	Signal Chain Installation:
19		2
20	1. Install numpy, matplotlib, TKAgg	3	1. Install system dependencies: python-pip python-dev gfortran libpng-dev freetype* libblas-dev liblapack-dev libatlas-base-dev python-qt4
21	2. Install digital_rf_hdf5 module (developed by Haystack Observatory)	4	2. Install digital_rf_hdf5 module (developed by Haystack Observatory)
22	if you want to use USRP data	5	if you want to use USRP data
23	3. untar schainpy-x.x.x.tar.gz	6	3. untar schainpy-x.x.x.tar.gz
24	4. cd schainpy-x.x.x	7	4. cd schainpy-x.x.x
25	5. execute:	8	5. execute:
26	[hostname]$ sudo p~~yhon setup.py~~ install	9	[hostname]$ sudo pip install ./
27	6. testing gui:	10	6. testing gui:
28	[hostname]$ schainGUI (enter)	11	[hostname]$ schainGUI (enter)
29		12

schainpy/controller.py +5 -5

                     self.parmConfObjList = []
-                    parmElementList = opElement.getiterator(ParameterConf().getElementName())
+                    parmElementList = opElement.iter(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                     self.opConfObjList = []
-                    opElementList = upElement.getiterator(OperationConf().getElementName())
+                    opElementList = upElement.iter(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                     self.opConfObjList = []
-                    opElementList = upElement.getiterator(OperationConf().getElementName())
+                    opElementList = upElement.iter(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                     self.name = self.projectElement.get('name')
                     self.description = self.projectElement.get('description')
-                    readUnitElementList = self.projectElement.getiterator(ReadUnitConf().getElementName())
+                    readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
                     for readUnitElement in readUnitElementList:
                         readUnitConfObj = ReadUnitConf()
                         self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
-                    procUnitElementList = self.projectElement.getiterator(ProcUnitConf().getElementName())
+                    procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
                     for procUnitElement in procUnitElementList:
                         procUnitConfObj = ProcUnitConf()

schainpy/model/graphics/figure.py +15 -11

                 decimationy = None
                 __MAXNUMX = 200
-                __MAXNUMY = 400
+                __MAXNUMY = 100
                 __MAXNUMTIME = 500
                     xlen = len(x)
                     ylen = len(y)
-                    decimationx = numpy.floor(xlen/self.__MAXNUMX) - 1 if numpy.floor(xlen/self.__MAXNUMX)>1 else 1
+                    decimationx = int(xlen/self.__MAXNUMX)+1 \
-                    decimationy = numpy.floor(ylen/self.__MAXNUMY) + 1
+                                  if int(xlen/self.__MAXNUMX)>1 else 1
+                    decimationy = int(ylen/self.__MAXNUMY) \
+                                  if int(ylen/self.__MAXNUMY)>1 else 1
+                    x_buffer = x#[::decimationx]
+                    y_buffer = y#[::decimationy]
+                    z_buffer = z#[::decimationx, ::decimationy]
-                    x_buffer = x[::decimationx]
-                    y_buffer = y[::decimationy]
-                    z_buffer = z[::decimationx, ::decimationy]
                     #===================================================
                     if self.__firsttime:
                     xlen = len(self.x_buffer)
                     ylen = len(y)
-                    decimationx = numpy.floor(xlen/maxNumX) + 1
+                    decimationx = int(xlen/self.__MAXNUMX) \
-                    decimationy = numpy.floor(ylen/maxNumY) + 1
+                                  if int(xlen/self.__MAXNUMX)>1 else 1
+                    decimationy = int(ylen/self.__MAXNUMY) \
+                                  if int(ylen/self.__MAXNUMY)>1 else 1
-                    x_buffer = self.x_buffer[::decimationx]
+                    x_buffer = self.x_buffer#[::decimationx]
-                    y_buffer = y[::decimationy]
+                    y_buffer = y#[::decimationy]
-                    z_buffer = z_buffer[::decimationx, ::decimationy]
+                    z_buffer = z_buffer#[::decimationx, ::decimationy]
                     #===================================================
                     x_buffer, y_buffer, z_buffer = self.__fillGaps(x_buffer, y_buffer, z_buffer)

schainpy/model/graphics/jroplot_parameters.py +14 -12

             from figure import Figure, isRealtime, isTimeInHourRange
             from plotting_codes import *
+            import matplotlib.pyplot as plt
             class MomentsPlot(Figure):
                 isConfig = None
             #         tmax = None
                     x = dataOut.getTimeRange1(dataOut.outputInterval)
-            #         y = dataOut.heightList
                     y = dataOut.heightList
                     z = dataOut.data_output.copy()
                     nplots = z.shape[0]    #Number of wind dimensions estimated
                     nplotsw = nplots
                     #If there is a SNR function defined
             #         showprofile = False
             #        thisDatetime = dataOut.datatime
-                    thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
+                    #thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
+                    thisDatetime = datetime.datetime.now()
                     title = wintitle + "Wind"
                     xlabel = ""
                     ylabel = "Height (km)"
                         self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
-                        if ymin == None: ymin = numpy.nanmin(y)
+                        #if ymin == None: ymin = numpy.nanmin(y)
-                        if ymax == None: ymax = numpy.nanmax(y)
+                        #if ymax == None: ymax = numpy.nanmax(y)
                         if zmax == None: zmax = numpy.nanmax(abs(z[range(2),:]))
                         #if numpy.isnan(zmax): zmax = 50
                             if zmax_ver == None: zmax_ver = numpy.nanmax(abs(z[2,:]))
                             if zmin_ver == None: zmin_ver = -zmax_ver
-                        if dataOut.data_SNR is not None:
+            #             if dataOut.data_SNR is not None:
-                            if SNRmin == None:  SNRmin = numpy.nanmin(SNRavgdB)
+            #                 if SNRmin == None:  SNRmin = numpy.nanmin(SNRavgdB)
-                            if SNRmax == None:  SNRmax = numpy.nanmax(SNRavgdB)
+            #                 if SNRmax == None:  SNRmax = numpy.nanmax(SNRavgdB)
                         self.FTP_WEI = ftp_wei
                     self.setWinTitle(title)
-                    if ((self.xmax - x[1]) < (x[1]-x[0])):
+                    #if ((self.xmax - x[1]) < (x[1]-x[0])):
-                        x[1] = self.xmax
+                    #    x[1] = self.xmax
                     strWind = ['Zonal', 'Meridional', 'Vertical']
                     strCb = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)']
                               thisDatetime=thisDatetime,
                               update_figfile=update_figfile)
-                    if dataOut.ltctime + dataOut.outputInterval >= self.xmax:
+                    if False and dataOut.ltctime + dataOut.outputInterval >= self.xmax:
                         self.counter_imagwr = wr_period
                         self.isConfig = False
                         update_figfile = True
-            class Parameters1Plot(Figure):
+            class ParametersPlot(Figure):
                 __isConfig = None
                 __nsubplots = None

schainpy/model/graphics/jroplot_spectra.py +6 -3

                         save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
                         server=None, folder=None, username=None, password=None,
                         ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
-                        xaxis="frequency", colormap='jet'):
+                        xaxis="velocity", **kwargs):
                     """
                         zmax            :    None
                     """
+                    colormap = kwargs.get('colormap','jet')
                     if realtime:
                         if not(isRealtime(utcdatatime = dataOut.utctime)):
                             print 'Skipping this plot function'
                 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
                         xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
-                        timerange=None, colormap='jet',
+                        timerange=None,
                         save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
                         server=None, folder=None, username=None, password=None,
-                        ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
+                        ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, **kwargs):
                     """
                         zmax            :    None
                     """
+                    colormap = kwargs.get('colormap', 'jet')
                     if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
                         return

schainpy/model/graphics/mpldriver.py +2 0

             #Actualizacion de las funciones del driver
             ###########################################
+            # create jro colormap
             jet_values = matplotlib.pyplot.get_cmap("jet", 100)(numpy.arange(100))[10:90]
             blu_values = matplotlib.pyplot.get_cmap("seismic_r", 20)(numpy.arange(20))[10:15]
             ncmap = matplotlib.colors.LinearSegmentedColormap.from_list("jro", numpy.vstack((blu_values, jet_values)))

schainpy/model/io/__init__.py 0 0

NO CONTENT: modified file

schainpy/model/io/jroIO_param.py 0 -63

@@ -399,70 +399,7 class ParamReader(ProcessingUnit):
399	self.listData = listdata	399	self.listData = listdata
400	return	400	return
401		401
402	def __setDataArray(self, dataset, shapes):
403		402
404	nDims = shapes[0]
405
406	nDim2 = shapes[1] #Dimension 0
407
408	nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
409
410	nDim0 = shapes[3] #Dimension 2, number of samples or ranges
411
412	mode = shapes[4] #Mode of storing
413
414	blockList = self.blockList
415
416	blocksPerFile = self.blocksPerFile
417
418	#Depending on what mode the data was stored
419	if mode == 0: #Divided in channels
420	arrayData = dataset.value.astype(numpy.float)[0][blockList]
421	if mode == 1: #Divided in parameter
422	strds = 'table'
423	nDatas = nDim1
424	newShapes = (blocksPerFile,nDim2,nDim0)
425	elif mode==2: #Concatenated in a table
426	strds = 'table0'
427	arrayData = dataset[strds].value
428	#Selecting part of the dataset
429	utctime = arrayData[:,0]
430	u, indices = numpy.unique(utctime, return_index=True)
431
432	if blockList.size != indices.size:
433	indMin = indices[blockList[0]]
434	if blockList[-1] + 1 >= indices.size:
435	arrayData = arrayData[indMin:,:]
436	else:
437	indMax = indices[blockList[-1] + 1]
438	arrayData = arrayData[indMin:indMax,:]
439	return arrayData
440
441	#------- One dimension ---------------
442	if nDims == 0:
443	arrayData = dataset.value.astype(numpy.float)[0][blockList]
444
445	#------- Two dimensions -----------
446	elif nDims == 2:
447	arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
448	newShapes = (blocksPerFile,nDim0)
449	nDatas = nDim1
450
451	for i in range(nDatas):
452	data = dataset[strds + str(i)].value
453	arrayData[:,i,:] = data[blockList,:]
454
455	#------- Three dimensions ---------
456	else:
457	arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
458	for i in range(nDatas):
459
460	data = dataset[strds + str(i)].value
461
462	for b in range(blockList.size):
463	arrayData[b,:,i,:] = data[:,:,blockList[b]]
464
465	return arrayData
466		403
467	def __setDataOut(self):	404	def __setDataOut(self):
468	listMeta = self.listMeta	405	listMeta = self.listMeta

schainpy/model/io/jroIO_spectra.py +2 0

                     if self.dataOut.flagDiscontinuousBlock:
                         self.data_spc.fill(0)
+                        if self.dataOut.data_cspc is not None:
                             self.data_cspc.fill(0)
+                        if self.dataOut.data_dc is not None:
                             self.data_dc.fill(0)
                         self.setNextFile()

schainpy/model/proc/jroproc_base.py 0 0

NO CONTENT: modified file

schainpy/model/proc/jroproc_heispectra.py +2 0

                     self.dataOut.nCode = self.dataIn.nCode
                     self.dataOut.code = self.dataIn.code
             #        self.dataOut.nProfiles = 1
+                    self.dataOut.ippFactor = 1
+                    self.dataOut.noise_estimation = None
             #        self.dataOut.nProfiles = self.dataOut.nFFTPoints
                     self.dataOut.nFFTPoints = self.dataIn.nHeights
             #        self.dataOut.channelIndexList = self.dataIn.channelIndexList

schainpy/model/utils/__init__.py +1 -1

             '''
             from jroutils_ftp import *
-            from jroutils_publish import PublishData
  No newline at end of file
+            from jroutils_publish import *

schainpy/model/utils/jroutils_publish.py +92 -42

@@ -9,11 +9,11 import paho.mqtt.client as mqtt
9		9
10	from schainpy.model.proc.jroproc_base import Operation	10	from schainpy.model.proc.jroproc_base import Operation
11		11
12
13	class PrettyFloat(float):	12	class PrettyFloat(float):
14	def __repr__(self):	13	def __repr__(self):
15	return '%.2f' % self	14	return '%.2f' % self
16		15
		16
17	def pretty_floats(obj):	17	def pretty_floats(obj):
18	if isinstance(obj, float):	18	if isinstance(obj, float):
19	return PrettyFloat(obj)	19	return PrettyFloat(obj)
@@ -23,80 +23,130 def pretty_floats(obj):
23	return map(pretty_floats, obj)	23	return map(pretty_floats, obj)
24	return obj	24	return obj
25		25
		26
26	class PublishData(Operation):	27	class PublishData(Operation):
		28	"""Clase publish."""
27		29
28	__MAXNUMX = 80	30	__MAXNUMX = 80
29	__MAXNUMY = 80	31	__MAXNUMY = 80
30		32
31	def __init__(self):	33	def __init__(self):
32		34	"""Inicio."""
33	Operation.__init__(self)	35	Operation.__init__(self)
34
35	self.isConfig = False	36	self.isConfig = False
36	self.client = None	37	self.client = None
37		38
38	@staticmethod	39	def on_disconnect(self, client, userdata, rc):
39	def on_disconnect(client, userdata, rc):
40	if rc != 0:	40	if rc != 0:
41	print("Unexpected disconnection.")	41	print("Unexpected disconnection.")
		42	self.connect()
42		43
43	def setup(self, host, port=1883, username=None, password=None, **kwargs):	44	def connect(self):
44		45	print 'trying to connect'
45	self.client = mqtt.Client()
46	try:	46	try:
47	self.client.connect(host=host, port=port, keepalive=60*10, bind_address='')	47	self.client.connect(
		48	host=self.host,
		49	port=self.port,
		50	keepalive=60*10,
		51	bind_address='')
		52	print "connected"
		53	self.client.loop_start()
		54	# self.client.publish(
		55	# self.topic + 'SETUP',
		56	# json.dumps(setup),
		57	# retain=True
		58	# )
48	except:	59	except:
		60	print "MQTT Conection error."
49	self.client = False	61	self.client = False
		62
		63	def setup(self, host, port=1883, username=None, password=None, **kwargs):
		64
50	self.topic = kwargs.get('topic', 'schain')	65	self.topic = kwargs.get('topic', 'schain')
51	self.delay = kwargs.get('delay', 0)	66	self.delay = kwargs.get('delay', 0)
		67	self.plottype = kwargs.get('plottype', 'spectra')
52	self.host = host	68	self.host = host
53	self.port = port	69	self.port = port
54	self.cnt = 0	70	self.cnt = 0
55		71	setup = []
56	def run(self, dataOut, host, datatype='data_spc', **kwargs):	72	for plot in self.plottype:
57		73	setup.append({
58	if not self.isConfig:	74	'plot': plot,
59	self.setup(host, **kwargs)	75	'topic': self.topic + plot,
60	self.isConfig = True	76	'title': getattr(self, plot + '_' + 'title', False),
61		77	'xlabel': getattr(self, plot + '_' + 'xlabel', False),
62	data = getattr(dataOut, datatype)	78	'ylabel': getattr(self, plot + '_' + 'ylabel', False),
63		79	'xrange': getattr(self, plot + '_' + 'xrange', False),
64	z = data/dataOut.normFactor	80	'yrange': getattr(self, plot + '_' + 'yrange', False),
		81	'zrange': getattr(self, plot + '_' + 'zrange', False),
		82	})
		83	self.client = mqtt.Client(
		84	client_id='jc'+self.topic + 'SCHAIN',
		85	clean_session=True)
		86	self.client.on_disconnect = self.on_disconnect
		87	self.connect()
		88
		89	def publish_data(self, plottype):
		90	data = getattr(self.dataOut, 'data_spc')
		91	if plottype == 'spectra':
		92	z = data/self.dataOut.normFactor
65	zdB = 10*numpy.log10(z)	93	zdB = 10*numpy.log10(z)
66	avg = numpy.average(z, axis=1)
67	avgdB = 10*numpy.log10(avg)
68
69	xlen ,ylen = zdB[0].shape	94	xlen, ylen = zdB[0].shape
70
71
72	dx = numpy.floor(xlen/self.__MAXNUMX) + 1	95	dx = numpy.floor(xlen/self.__MAXNUMX) + 1
73	dy = numpy.floor(ylen/self.__MAXNUMY) + 1	96	dy = numpy.floor(ylen/self.__MAXNUMY) + 1
74		97	Z = [0 for i in self.dataOut.channelList]
75	Z = [0 for i in dataOut.channelList]	98	for i in self.dataOut.channelList:
76	AVG = [0 for i in dataOut.channelList]
77
78	for i in dataOut.channelList:
79	Z[i] = zdB[i][::dx, ::dy].tolist()	99	Z[i] = zdB[i][::dx, ::dy].tolist()
80	AVG[i] = avgdB[i][::dy].tolist()	100	payload = {
81		101	'timestamp': self.dataOut.utctime,
82	payload = {'timestamp':dataOut.utctime,
83	'data':pretty_floats(Z),	102	'data': pretty_floats(Z),
84	'data_profile':pretty_floats(AVG),	103	'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
85	'channels': ['Ch %s' % ch for ch in dataOut.channelList],	104	'interval': self.dataOut.getTimeInterval(),
86	'interval': dataOut.getTimeInterval(),	105	'xRange': [0, 80]
87	}	106	}
88		107
		108	elif plottype in ('rti', 'power'):
		109	z = data/self.dataOut.normFactor
		110	avg = numpy.average(z, axis=1)
		111	avgdB = 10*numpy.log10(avg)
		112	xlen, ylen = z[0].shape
		113	dy = numpy.floor(ylen/self.__MAXNUMY) + 1
		114	AVG = [0 for i in self.dataOut.channelList]
		115	for i in self.dataOut.channelList:
		116	AVG[i] = avgdB[i][::dy].tolist()
		117	payload = {
		118	'timestamp': self.dataOut.utctime,
		119	'data': pretty_floats(AVG),
		120	'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
		121	'interval': self.dataOut.getTimeInterval(),
		122	'xRange': [0, 80]
		123	}
		124	elif plottype == 'noise':
		125	noise = self.dataOut.getNoise()/self.dataOut.normFactor
		126	noisedB = 10*numpy.log10(noise)
		127	payload = {
		128	'timestamp': self.dataOut.utctime,
		129	'data': pretty_floats(noisedB.reshape(-1, 1).tolist()),
		130	'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
		131	'interval': self.dataOut.getTimeInterval(),
		132	'xRange': [0, 80]
		133	}
89		134
90	#if self.cnt==self.interval and self.client:
91	print 'Publishing data to {}'.format(self.host)	135	print 'Publishing data to {}'.format(self.host)
92	self.client.publish(self.topic, json.dumps(payload), qos=0)	136	print '*************************'
93	time.sleep(self.delay)	137	self.client.publish(self.topic + plottype, json.dumps(payload), qos=0)
94	#self.cnt = 0
95	#else:
96	# self.cnt += 1
97		138
98		139
99	def close(self):	140	def run(self, dataOut, host, **kwargs):
		141	self.dataOut = dataOut
		142	if not self.isConfig:
		143	self.setup(host, **kwargs)
		144	self.isConfig = True
100		145
		146	map(self.publish_data, self.plottype)
		147	time.sleep(self.delay)
		148
		149	def close(self):
101	if self.client:	150	if self.client:
		151	self.client.loop_stop()
102	self.client.disconnect()	152	self.client.disconnect()

setup.py +5 -1

                     include_package_data=False,
                     scripts =['schainpy/gui/schainGUI',
                               'schainpy/scripts/schain'],
-                    install_requires=["numpy >= 1.6.0",
+                    install_requires=[
                                       "scipy >= 0.9.0",
+                                      "h5py >= 2.0.1",
                                       "matplotlib >= 1.0.0",
+                                      "pyfits >= 2.0.0",
+                                      "numpy >= 1.6.0",
+                                      "paramiko",
                                       ],
                   )

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages