schain-jc Commit - r1164:80b8ab8c0c11

Merge with px1000 branch

Juan C. Espinoza -

r1164:80b8ab8c0c11

parent child

Context file:

r1164:80b8ab8c0c11

Collapse all files

schainpy/model/io/pxIO_param.py created 644 +350 0

@@ -0,0 +1,350
	1	'''
	2	Created on Jan 15, 2018
	3
	4	@author: Juan C. Espinoza
	5	'''
	6
	7	import os
	8	import sys
	9	import time
	10	import glob
	11	import datetime
	12	import tarfile
	13
	14	import numpy
	15	try:
	16	from netCDF4 import Dataset
	17	except:
	18	log.warning(
	19	'You should install "netCDF4" module if you want to read/write NCDF files'
	20	)
	21
	22	from utils import folder_in_range
	23
	24	from schainpy.model.io.jroIO_base import JRODataReader
	25	from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
	26	from schainpy.model.data.jrodata import Parameters
	27	from schainpy.utils import log
	28
	29	UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
	30
	31
	32	class PXReader(JRODataReader, ProcessingUnit):
	33
	34	def __init__(self, **kwargs):
	35
	36	ProcessingUnit.__init__(self, **kwargs)
	37
	38	self.dataOut = Parameters()
	39	self.counter_records = 0
	40	self.nrecords = None
	41	self.flagNoMoreFiles = 0
	42	self.isConfig = False
	43	self.filename = None
	44	self.intervals = set()
	45	self.ext = ('.nc', '.tgz')
	46	self.online_mode = False
	47
	48	def setup(self,
	49	path=None,
	50	startDate=None,
	51	endDate=None,
	52	format=None,
	53	startTime=datetime.time(0, 0, 0),
	54	endTime=datetime.time(23, 59, 59),
	55	walk=False,
	56	**kwargs):
	57
	58	self.path = path
	59	self.startDate = startDate
	60	self.endDate = endDate
	61	self.startTime = startTime
	62	self.endTime = endTime
	63	self.datatime = datetime.datetime(1900,1,1)
	64	self.walk = walk
	65	self.nTries = kwargs.get('nTries', 10)
	66	self.online = kwargs.get('online', False)
	67	self.delay = kwargs.get('delay', 60)
	68	self.ele = kwargs.get('ext', '')
	69
	70	if self.path is None:
	71	raise ValueError, 'The path is not valid'
	72
	73	self.search_files(path, startDate, endDate, startTime, endTime, walk)
	74	self.cursor = 0
	75	self.counter_records = 0
	76
	77	if not self.files:
	78	raise Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path)
	79
	80	def search_files(self, path, startDate, endDate, startTime, endTime, walk):
	81	'''
	82	Searching for NCDF files in path
	83	Creating a list of files to procces included in [startDate,endDate]
	84
	85	Input:
	86	path - Path to find files
	87	'''
	88
	89	log.log('Searching files {} in {} '.format(self.ext, path), 'PXReader')
	90	if walk:
	91	paths = [os.path.join(path, p) for p in os.listdir(path) if os.path.isdir(os.path.join(path, p))]
	92	paths.sort()
	93	else:
	94	paths = [path]
	95
	96	fileList0 = []
	97
	98	for subpath in paths:
	99	if not folder_in_range(subpath.split('/')[-1], startDate, endDate, '%Y%m%d'):
	100	continue
	101	fileList0 += [os.path.join(subpath, s) for s in glob.glob1(subpath, '*') if os.path.splitext(s)[-1] in self.ext and '{}'.format(self.ele) in s]
	102
	103	fileList0.sort()
	104	if self.online:
	105	fileList0 = fileList0[-1:]
	106
	107	self.files = {}
	108
	109	startDate = startDate - datetime.timedelta(1)
	110	endDate = endDate + datetime.timedelta(1)
	111
	112	for fullname in fileList0:
	113	thisFile = fullname.split('/')[-1]
	114	year = thisFile[3:7]
	115	if not year.isdigit():
	116	continue
	117
	118	month = thisFile[7:9]
	119	if not month.isdigit():
	120	continue
	121
	122	day = thisFile[9:11]
	123	if not day.isdigit():
	124	continue
	125
	126	year, month, day = int(year), int(month), int(day)
	127	dateFile = datetime.date(year, month, day)
	128	timeFile = datetime.time(int(thisFile[12:14]), int(thisFile[14:16]), int(thisFile[16:18]))
	129
	130	if (startDate > dateFile) or (endDate < dateFile):
	131	continue
	132
	133	dt = datetime.datetime.combine(dateFile, timeFile)
	134	if dt not in self.files:
	135	self.files[dt] = []
	136	self.files[dt].append(fullname)
	137
	138	self.dates = self.files.keys()
	139	self.dates.sort()
	140
	141	return
	142
	143	def search_files_online(self):
	144	'''
	145	Searching for NCDF files in online mode path
	146	Creating a list of files to procces included in [startDate,endDate]
	147
	148	Input:
	149	path - Path to find files
	150	'''
	151
	152	self.files = {}
	153
	154	for n in range(self.nTries):
	155
	156	if self.walk:
	157	paths = [os.path.join(self.path, p) for p in os.listdir(self.path) if os.path.isdir(os.path.join(self.path, p))]
	158	paths.sort()
	159	path = paths[-1]
	160	else:
	161	path = self.path
	162
	163	new_files = [os.path.join(path, s) for s in glob.glob1(path, '*') if os.path.splitext(s)[-1] in self.ext and '{}'.format(self.ele) in s]
	164	new_files.sort()
	165
	166	for fullname in new_files:
	167	thisFile = fullname.split('/')[-1]
	168	year = thisFile[3:7]
	169	if not year.isdigit():
	170	continue
	171
	172	month = thisFile[7:9]
	173	if not month.isdigit():
	174	continue
	175
	176	day = thisFile[9:11]
	177	if not day.isdigit():
	178	continue
	179
	180	year, month, day = int(year), int(month), int(day)
	181	dateFile = datetime.date(year, month, day)
	182	timeFile = datetime.time(int(thisFile[12:14]), int(thisFile[14:16]), int(thisFile[16:18]))
	183
	184	dt = datetime.datetime.combine(dateFile, timeFile)
	185
	186	if self.dt >= dt:
	187	continue
	188
	189	if dt not in self.files:
	190	self.dt = dt
	191	self.files[dt] = []
	192
	193	self.files[dt].append(fullname)
	194	break
	195
	196	if self.files:
	197	break
	198	else:
	199	log.warning('Waiting {} seconds for the next file, try {} ...'.format(self.delay, n + 1), 'PXReader')
	200	time.sleep(self.delay)
	201
	202	if not self.files:
	203	return 0
	204
	205	self.dates = self.files.keys()
	206	self.dates.sort()
	207	self.cursor = 0
	208
	209	return 1
	210
	211	def parseFile(self):
	212	'''
	213	'''
	214
	215	header = {}
	216
	217	for attr in self.fp.ncattrs():
	218	header[str(attr)] = getattr(self.fp, attr)
	219
	220	self.header.append(header)
	221
	222	self.data[header['TypeName']] = numpy.array(self.fp.variables[header['TypeName']])
	223
	224	def setNextFile(self):
	225	'''
	226	Open next files for the current datetime
	227	'''
	228
	229	cursor = self.cursor
	230	if not self.online_mode:
	231	if cursor == len(self.dates):
	232	if self.online:
	233	cursor = 0
	234	self.dt = self.dates[cursor]
	235	self.online_mode = True
	236	if not self.search_files_online():
	237	log.success('No more files', 'PXReader')
	238	return 0
	239	else:
	240	log.success('No more files', 'PXReader')
	241	self.flagNoMoreFiles = 1
	242	return 0
	243	else:
	244	if not self.search_files_online():
	245	return 0
	246	cursor = self.cursor
	247
	248	self.data = {}
	249	self.header = []
	250
	251	for fullname in self.files[self.dates[cursor]]:
	252
	253	log.log('Opening: {}'.format(fullname), 'PXReader')
	254
	255	if os.path.splitext(fullname)[-1] == '.tgz':
	256	tar = tarfile.open(fullname, 'r:gz')
	257	tar.extractall('/tmp')
	258	files = [os.path.join('/tmp', member.name) for member in tar.getmembers()]
	259	else:
	260	files = [fullname]
	261
	262	for filename in files:
	263	if self.filename is not None:
	264	self.fp.close()
	265
	266	self.filename = filename
	267	self.filedate = self.dates[cursor]
	268	self.fp = Dataset(self.filename, 'r')
	269	self.parseFile()
	270
	271	self.counter_records += 1
	272	self.cursor += 1
	273	return 1
	274
	275	def readNextFile(self):
	276
	277	while True:
	278	self.flagDiscontinuousBlock = 0
	279	if not self.setNextFile():
	280	return 0
	281
	282	self.datatime = datetime.datetime.utcfromtimestamp(self.header[0]['Time'])
	283
	284	if self.online:
	285	break
	286
	287	if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
	288	(self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
	289	log.warning(
	290	'Reading Record No. {}/{} -> {} [Skipping]'.format(
	291	self.counter_records,
	292	self.nrecords,
	293	self.datatime.ctime()),
	294	'PXReader')
	295	continue
	296	break
	297
	298	log.log(
	299	'Reading Record No. {}/{} -> {}'.format(
	300	self.counter_records,
	301	self.nrecords,
	302	self.datatime.ctime()),
	303	'PXReader')
	304
	305	return 1
	306
	307
	308	def set_output(self):
	309	'''
	310	Storing data from buffer to dataOut object
	311	'''
	312
	313	self.data['Elevation'] = numpy.array(self.fp.variables['Elevation'])
	314	self.data['Azimuth'] = numpy.array(self.fp.variables['Azimuth'])
	315	self.dataOut.range = numpy.array(self.fp.variables['GateWidth'])
	316	self.dataOut.data = self.data
	317	self.dataOut.units = [h['Unit-value'] for h in self.header]
	318	self.dataOut.parameters = [h['TypeName'] for h in self.header]
	319	self.dataOut.missing = self.header[0]['MissingData']
	320	self.dataOut.max_range = self.header[0]['MaximumRange-value']
	321	self.dataOut.elevation = self.header[0]['Elevation']
	322	self.dataOut.azimuth = self.header[0]['Azimuth']
	323	self.dataOut.latitude = self.header[0]['Latitude']
	324	self.dataOut.longitude = self.header[0]['Longitude']
	325	self.dataOut.utctime = self.header[0]['Time']
	326	self.dataOut.utctimeInit = self.dataOut.utctime
	327	self.dataOut.useLocalTime = True
	328	self.dataOut.flagNoData = False
	329	self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
	330
	331	log.log('Parameters found: {}'.format(','.join(self.dataOut.parameters)),
	332	'PXReader')
	333
	334	def getData(self):
	335	'''
	336	Storing data from databuffer to dataOut object
	337	'''
	338	if self.flagNoMoreFiles:
	339	self.dataOut.flagNoData = True
	340	log.error('No file left to process', 'PXReader')
	341	return 0
	342
	343	if not self.readNextFile():
	344	self.dataOut.flagNoData = True
	345	return 0
	346
	347	self.set_output()
	348
	349	return 1
	350

schainpy/model/io/utils.py created 644 +24 0

@@ -0,0 +1,24
	1	"""
	2	Utilities for IO modules
	3	"""
	4
	5	import os
	6	from datetime import datetime
	7
	8	def folder_in_range(folder, start_date, end_date, pattern):
	9	"""
	10	Check whether folder is bettwen start_date and end_date
	11
	12	Args:
	13	folder (str): Folder to check
	14	start_date (date): Initial date
	15	end_date (date): Final date
	16	pattern (str): Datetime format of the folder
	17	Returns:
	18	bool: True for success, False otherwise
	19	"""
	20	try:
	21	dt = datetime.strptime(folder, pattern)
	22	except:
	23	raise ValueError('Folder {} does not match {} format'.format(folder, pattern))
	24	return start_date <= dt.date() <= end_date

schainpy/model/proc/pxproc_parameters.py created 644 +64 0

@@ -0,0 +1,64
	1	'''
	2	Created on Oct 24, 2016
	3
	4	@author: roj- LouVD
	5	'''
	6
	7	import numpy
	8	import datetime
	9	import time
	10	from time import gmtime
	11
	12	from numpy import transpose
	13
	14	from jroproc_base import ProcessingUnit, Operation
	15	from schainpy.model.data.jrodata import Parameters
	16
	17
	18	class PXParametersProc(ProcessingUnit):
	19	'''
	20	Processing unit for PX parameters data
	21	'''
	22
	23	def __init__(self, **kwargs):
	24	"""
	25	Inputs: None
	26	"""
	27	ProcessingUnit.__init__(self, **kwargs)
	28	self.dataOut = Parameters()
	29	self.isConfig = False
	30
	31	def setup(self, mode):
	32	"""
	33	"""
	34	self.dataOut.mode = mode
	35
	36	def run(self, mode):
	37	"""
	38	Args:
	39	mode (str): select independent variable 'E' for elevation or 'A' for azimuth
	40	"""
	41
	42	if not self.isConfig:
	43	self.setup(mode)
	44	self.isConfig = True
	45
	46	if self.dataIn.type == 'Parameters':
	47	self.dataOut.copy(self.dataIn)
	48
	49	self.dataOut.data_param = numpy.array([self.dataOut.data[var] for var in self.dataOut.parameters])
	50	self.dataOut.data_param[self.dataOut.data_param == self.dataOut.missing] = numpy.nan
	51
	52	if mode.upper()=='E':
	53	self.dataOut.heightList = self.dataOut.data['Azimuth']
	54	else:
	55	self.dataOut.heightList = self.dataOut.data['Elevation']
	56
	57	attrs = ['units', 'elevation', 'azimuth', 'max_range', 'latitude', 'longitude']
	58	meta = {}
	59
	60	for attr in attrs:
	61	meta[attr] = getattr(self.dataOut, attr)
	62
	63	meta['mode'] = mode
	64	self.dataOut.meta = meta No newline at end of file

schainpy/controller.py +1 -1

                 def run(self):
-                    log.success('Starting {}'.format(self.name))
+                    log.success('Starting {}'.format(self.name), tag='')
                     self.start_time = time.time()
                     self.createObjects()
                     self.connectObjects()

schainpy/model/graphics/jroplot_data.py +180 -12

@@ -9,6 +9,7 import zmq
9	import numpy	9	import numpy
10	import matplotlib	10	import matplotlib
11	import matplotlib.pyplot as plt	11	import matplotlib.pyplot as plt
		12	from matplotlib.patches import Polygon
12	from mpl_toolkits.axes_grid1 import make_axes_locatable	13	from mpl_toolkits.axes_grid1 import make_axes_locatable
13	from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator	14	from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
14		15
@@ -24,6 +25,23 matplotlib.pyplot.register_cmap(cmap=ncmap)
24		25
25	CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis', 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]	26	CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis', 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
26		27
		28	EARTH_RADIUS = 6.3710e3
		29
		30	def ll2xy(lat1, lon1, lat2, lon2):
		31
		32	p = 0.017453292519943295
		33	a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
		34	r = 12742 * numpy.arcsin(numpy.sqrt(a))
		35	theta = numpy.arctan2(numpy.sin((lon2-lon1)p)numpy.cos(lat2p), numpy.cos(lat1p)numpy.sin(lat2p)-numpy.sin(lat1p)numpy.cos(lat2p)numpy.cos((lon2-lon1)*p))
		36	theta = -theta + numpy.pi/2
		37	return rnumpy.cos(theta), rnumpy.sin(theta)
		38
		39	def km2deg(km):
		40	'''
		41	Convert distance in km to degrees
		42	'''
		43
		44	return numpy.rad2deg(km/EARTH_RADIUS)
27		45
28	def figpause(interval):	46	def figpause(interval):
29	backend = plt.rcParams['backend']	47	backend = plt.rcParams['backend']
@@ -64,7 +82,7 class PlotData(Operation, Process):
64	__attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',	82	__attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
65	'zlimits', 'xlabel', 'ylabel', 'xaxis','cb_label', 'title',	83	'zlimits', 'xlabel', 'ylabel', 'xaxis','cb_label', 'title',
66	'colorbar', 'bgcolor', 'width', 'height', 'localtime', 'oneFigure',	84	'colorbar', 'bgcolor', 'width', 'height', 'localtime', 'oneFigure',
67	'showprofile', 'decimation']	85	'showprofile', 'decimation', 'ftp']
68		86
69	def __init__(self, **kwargs):	87	def __init__(self, **kwargs):
70		88
@@ -81,6 +99,7 class PlotData(Operation, Process):
81	self.localtime = kwargs.pop('localtime', True)	99	self.localtime = kwargs.pop('localtime', True)
82	self.show = kwargs.get('show', True)	100	self.show = kwargs.get('show', True)
83	self.save = kwargs.get('save', False)	101	self.save = kwargs.get('save', False)
		102	self.ftp = kwargs.get('ftp', False)
84	self.colormap = kwargs.get('colormap', self.colormap)	103	self.colormap = kwargs.get('colormap', self.colormap)
85	self.colormap_coh = kwargs.get('colormap_coh', 'jet')	104	self.colormap_coh = kwargs.get('colormap_coh', 'jet')
86	self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')	105	self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
@@ -90,6 +109,7 class PlotData(Operation, Process):
90	self.title = kwargs.get('wintitle', self.CODE.upper())	109	self.title = kwargs.get('wintitle', self.CODE.upper())
91	self.cb_label = kwargs.get('cb_label', None)	110	self.cb_label = kwargs.get('cb_label', None)
92	self.cb_labels = kwargs.get('cb_labels', None)	111	self.cb_labels = kwargs.get('cb_labels', None)
		112	self.labels = kwargs.get('labels', None)
93	self.xaxis = kwargs.get('xaxis', 'frequency')	113	self.xaxis = kwargs.get('xaxis', 'frequency')
94	self.zmin = kwargs.get('zmin', None)	114	self.zmin = kwargs.get('zmin', None)
95	self.zmax = kwargs.get('zmax', None)	115	self.zmax = kwargs.get('zmax', None)
@@ -97,8 +117,10 class PlotData(Operation, Process):
97	self.xmin = kwargs.get('xmin', None)	117	self.xmin = kwargs.get('xmin', None)
98	self.xmax = kwargs.get('xmax', None)	118	self.xmax = kwargs.get('xmax', None)
99	self.xrange = kwargs.get('xrange', 24)	119	self.xrange = kwargs.get('xrange', 24)
		120	self.xscale = kwargs.get('xscale', None)
100	self.ymin = kwargs.get('ymin', None)	121	self.ymin = kwargs.get('ymin', None)
101	self.ymax = kwargs.get('ymax', None)	122	self.ymax = kwargs.get('ymax', None)
		123	self.yscale = kwargs.get('yscale', None)
102	self.xlabel = kwargs.get('xlabel', None)	124	self.xlabel = kwargs.get('xlabel', None)
103	self.decimation = kwargs.get('decimation', None)	125	self.decimation = kwargs.get('decimation', None)
104	self.showSNR = kwargs.get('showSNR', False)	126	self.showSNR = kwargs.get('showSNR', False)
@@ -107,8 +129,10 class PlotData(Operation, Process):
107	self.height = kwargs.get('height', None)	129	self.height = kwargs.get('height', None)
108	self.colorbar = kwargs.get('colorbar', True)	130	self.colorbar = kwargs.get('colorbar', True)
109	self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])	131	self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
		132	self.channels = kwargs.get('channels', None)
110	self.titles = kwargs.get('titles', [])	133	self.titles = kwargs.get('titles', [])
111	self.polar = False	134	self.polar = False
		135	self.grid = kwargs.get('grid', False)
112		136
113	def __fmtTime(self, x, pos):	137	def __fmtTime(self, x, pos):
114	'''	138	'''
@@ -381,14 +405,19 class PlotData(Operation, Process):
381	ymin = self.ymin if self.ymin else numpy.nanmin(self.y)	405	ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
382	ymax = self.ymax if self.ymax else numpy.nanmax(self.y)	406	ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
383		407
384	Y = numpy.array([5, 10, 20, 50, 100, 200, 500, 1000, 2000])	408	Y = numpy.array([1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000])
385	i = 1 if numpy.where(ymax-ymin < Y)[0][0] < 0 else numpy.where(ymax-ymin < Y)[0][0]	409	i = 1 if numpy.where(abs(ymax-ymin) <= Y)[0][0] < 0 else numpy.where(abs(ymax-ymin) <= Y)[0][0]
386	ystep = Y[i] / 5	410	ystep = Y[i] / 10.
387		411
388	for n, ax in enumerate(self.axes):	412	for n, ax in enumerate(self.axes):
389	if ax.firsttime:	413	if ax.firsttime:
390	ax.set_facecolor(self.bgcolor)	414	ax.set_facecolor(self.bgcolor)
391	ax.yaxis.set_major_locator(MultipleLocator(ystep))	415	ax.yaxis.set_major_locator(MultipleLocator(ystep))
		416	ax.xaxis.set_major_locator(MultipleLocator(ystep))
		417	if self.xscale:
		418	ax.xaxis.set_major_formatter(FuncFormatter(lambda x, pos: '{0:g}'.format(x*self.xscale)))
		419	if self.xscale:
		420	ax.yaxis.set_major_formatter(FuncFormatter(lambda x, pos: '{0:g}'.format(x*self.yscale)))
392	if self.xaxis is 'time':	421	if self.xaxis is 'time':
393	ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))	422	ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
394	ax.xaxis.set_major_locator(LinearLocator(9))	423	ax.xaxis.set_major_locator(LinearLocator(9))
@@ -414,13 +443,15 class PlotData(Operation, Process):
414	ax.cbar.set_label(self.cb_labels[n], size=8)	443	ax.cbar.set_label(self.cb_labels[n], size=8)
415	else:	444	else:
416	ax.cbar = None	445	ax.cbar = None
		446	if self.grid:
		447	ax.grid(True)
417		448
418	if not self.polar:	449	if not self.polar:
419	ax.set_xlim(xmin, xmax)	450	ax.set_xlim(xmin, xmax)
420	ax.set_ylim(ymin, ymax)	451	ax.set_ylim(ymin, ymax)
421	ax.set_title('{} - {} {}'.format(	452	ax.set_title('{} {} {}'.format(
422	self.titles[n],	453	self.titles[n],
423	self.getDateTime(self.max_time).strftime('%H:%M:%S'),	454	self.getDateTime(self.max_time).strftime('%Y-%m-%dT%H:%M:%S'),
424	self.time_label),	455	self.time_label),
425	size=8)	456	size=8)
426	else:	457	else:
@@ -432,13 +463,15 class PlotData(Operation, Process):
432	def __plot(self):	463	def __plot(self):
433	'''	464	'''
434	'''	465	'''
435	log.~~success~~('Plotting', self.name)	466	log.log('Plotting', self.name)
436		467
437	try:	468	try:
438	self.plot()	469	self.plot()
439	self.format()	470	self.format()
440	except:	471	except Exception as e:
441	log.warning('{} Plot could not be updated... check data'.format(self.CODE), self.name)	472	log.warning('{} Plot could not be updated... check data'.format(self.CODE), self.name)
		473	log.error(str(e), '')
		474	return
442		475
443	for n, fig in enumerate(self.figures):	476	for n, fig in enumerate(self.figures):
444	if self.nrows == 0 or self.nplots == 0:	477	if self.nrows == 0 or self.nplots == 0:
@@ -452,14 +485,20 class PlotData(Operation, Process):
452	self.getDateTime(self.max_time).strftime('%Y/%m/%d')))	485	self.getDateTime(self.max_time).strftime('%Y/%m/%d')))
453	fig.canvas.draw()	486	fig.canvas.draw()
454		487
455	if self.save and self.data.ended:	488	if self.save and (self.data.ended or not self.data.buffering):
456	channels = range(self.nrows)	489
		490	if self.save_labels:
		491	labels = self.save_labels
		492	else:
		493	labels = range(self.nrows)
		494
457	if self.oneFigure:	495	if self.oneFigure:
458	label = ''	496	label = ''
459	else:	497	else:
460	label = '_{}'.format(~~chann~~els[n])	498	label = '-{}'.format(labels[n])
461	figname = os.path.join(	499	figname = os.path.join(
462	self.save,	500	self.save,
		501	self.CODE,
463	'{}{}_{}.png'.format(	502	'{}{}_{}.png'.format(
464	self.CODE,	503	self.CODE,
465	label,	504	label,
@@ -468,6 +507,8 class PlotData(Operation, Process):
468	)	507	)
469	)	508	)
470	log.log('Saving figure: {}'.format(figname), self.name)	509	log.log('Saving figure: {}'.format(figname), self.name)
		510	if not os.path.isdir(os.path.dirname(figname)):
		511	os.makedirs(os.path.dirname(figname))
471	fig.savefig(figname)	512	fig.savefig(figname)
472		513
473	def plot(self):	514	def plot(self):
@@ -477,7 +518,7 class PlotData(Operation, Process):
477		518
478	def run(self):	519	def run(self):
479		520
480	log.~~success~~('Starting', self.name)	521	log.log('Starting', self.name)
481		522
482	context = zmq.Context()	523	context = zmq.Context()
483	receiver = context.socket(zmq.SUB)	524	receiver = context.socket(zmq.SUB)
@@ -978,3 +1019,130 class PlotOutputData(PlotParamData):
978		1019
979	CODE = 'output'	1020	CODE = 'output'
980	colormap = 'seismic'	1021	colormap = 'seismic'
		1022
		1023
		1024	class PlotPolarMapData(PlotData):
		1025	'''
		1026	Plot for meteors detection data
		1027	'''
		1028
		1029	CODE = 'param'
		1030	colormap = 'seismic'
		1031
		1032	def setup(self):
		1033	self.ncols = 1
		1034	self.nrows = 1
		1035	self.width = 9
		1036	self.height = 8
		1037	self.mode = self.data.meta['mode']
		1038	if self.channels is not None:
		1039	self.nplots = len(self.channels)
		1040	self.nrows = len(self.channels)
		1041	else:
		1042	self.nplots = self.data.shape(self.CODE)[0]
		1043	self.nrows = self.nplots
		1044	self.channels = range(self.nplots)
		1045	if self.mode == 'E':
		1046	self.xlabel = 'Longitude'
		1047	self.ylabel = 'Latitude'
		1048	else:
		1049	self.xlabel = 'Range (km)'
		1050	self.ylabel = 'Height (km)'
		1051	self.bgcolor = 'white'
		1052	self.cb_labels = self.data.meta['units']
		1053	self.lat = self.data.meta['latitude']
		1054	self.lon = self.data.meta['longitude']
		1055	self.xmin, self.xmax = float(km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
		1056	self.ymin, self.ymax = float(km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
		1057	# self.polar = True
		1058
		1059	def plot(self):
		1060
		1061	for n, ax in enumerate(self.axes):
		1062	data = self.data['param'][self.channels[n]]
		1063
		1064	zeniths = numpy.linspace(0, self.data.meta['max_range'], data.shape[1])
		1065	if self.mode == 'E':
		1066	azimuths = -numpy.radians(self.data.heights)+numpy.pi/2
		1067	r, theta = numpy.meshgrid(zeniths, azimuths)
		1068	x, y = rnumpy.cos(theta)numpy.cos(numpy.radians(self.data.meta['elevation'])), rnumpy.sin(theta)numpy.cos(numpy.radians(self.data.meta['elevation']))
		1069	x = km2deg(x) + self.lon
		1070	y = km2deg(y) + self.lat
		1071	else:
		1072	azimuths = numpy.radians(self.data.heights)
		1073	r, theta = numpy.meshgrid(zeniths, azimuths)
		1074	x, y = rnumpy.cos(theta), rnumpy.sin(theta)
		1075	self.y = zeniths
		1076
		1077	if ax.firsttime:
		1078	if self.zlimits is not None:
		1079	self.zmin, self.zmax = self.zlimits[n]
		1080	ax.plt = ax.pcolormesh(#r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
		1081	x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
		1082	vmin=self.zmin,
		1083	vmax=self.zmax,
		1084	cmap=self.cmaps[n])
		1085	else:
		1086	if self.zlimits is not None:
		1087	self.zmin, self.zmax = self.zlimits[n]
		1088	ax.collections.remove(ax.collections[0])
		1089	ax.plt = ax.pcolormesh(# r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
		1090	x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
		1091	vmin=self.zmin,
		1092	vmax=self.zmax,
		1093	cmap=self.cmaps[n])
		1094
		1095	if self.mode == 'A':
		1096	continue
		1097
		1098	# plot district names
		1099	f = open('/data/workspace/schain_scripts/distrito.csv')
		1100	for line in f:
		1101	label, lon, lat = [s.strip() for s in line.split(',') if s]
		1102	lat = float(lat)
		1103	lon = float(lon)
		1104	# ax.plot(lon, lat, '.b', ms=2)
		1105	ax.text(lon, lat, label.decode('utf8'), ha='center', va='bottom', size='8', color='black')
		1106
		1107	# plot limites
		1108	limites =[]
		1109	tmp = []
		1110	for line in open('/data/workspace/schain_scripts/lima.csv'):
		1111	if '#' in line:
		1112	if tmp:
		1113	limites.append(tmp)
		1114	tmp = []
		1115	continue
		1116	values = line.strip().split(',')
		1117	tmp.append((float(values[0]), float(values[1])))
		1118	for points in limites:
		1119	ax.add_patch(Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
		1120
		1121	# plot Cuencas
		1122	for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
		1123	f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
		1124	values = [line.strip().split(',') for line in f]
		1125	points = [(float(s[0]), float(s[1])) for s in values]
		1126	ax.add_patch(Polygon(points, ec='b', fc='none'))
		1127
		1128	# plot grid
		1129	for r in (15, 30, 45, 60):
		1130	ax.add_artist(plt.Circle((self.lon, self.lat), km2deg(r), color='0.6', fill=False, lw=0.2))
		1131	ax.text(
		1132	self.lon + (km2deg(r))numpy.cos(60numpy.pi/180),
		1133	self.lat + (km2deg(r))numpy.sin(60numpy.pi/180),
		1134	'{}km'.format(r),
		1135	ha='center', va='bottom', size='8', color='0.6', weight='heavy')
		1136
		1137	if self.mode == 'E':
		1138	title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
		1139	label = 'E{:02d}'.format(int(self.data.meta['elevation']))
		1140	else:
		1141	title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
		1142	label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
		1143
		1144	self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
		1145	self.titles = ['{} {}'.format(self.data.parameters[x], title) for x in self.channels]
		1146	self.saveTime = self.max_time
		1147
		1148

schainpy/model/graphics/jroplot_spectra.py +2 -2

                     self.isConfig = False
                     self.__nsubplots = 1
-                    self.WIDTH = 300
+                    self.WIDTH = 250
-                    self.HEIGHT = 300
+                    self.HEIGHT = 250
                     self.WIDTHPROF = 120
                     self.HEIGHTPROF = 0
                     self.counter_imagwr = 0

schainpy/model/io/__init__.py +3 -1

             from bltrIO_param import *
             from jroIO_bltr import *
             from jroIO_mira35c import *
-            from julIO_param import *
  No newline at end of file
+            from julIO_param import *
+            from pxIO_param import *
  No newline at end of file

schainpy/model/proc/__init__.py +1 0

             from jroproc_spectra_lags import *
             from jroproc_spectra_acf import *
             from bltrproc_parameters import *
+            from pxproc_parameters import *

schainpy/model/utils/jroutils_publish.py +231 -28

@@ -2,12 +2,15
2	@author: Juan C. Espinoza	2	@author: Juan C. Espinoza
3	'''	3	'''
4		4
		5	import os
		6	import glob
5	import time	7	import time
6	import json	8	import json
7	import numpy	9	import numpy
8	import paho.mqtt.client as mqtt	10	import paho.mqtt.client as mqtt
9	import zmq	11	import zmq
10	import datetime	12	import datetime
		13	import ftplib
11	from zmq.utils.monitor import recv_monitor_message	14	from zmq.utils.monitor import recv_monitor_message
12	from functools import wraps	15	from functools import wraps
13	from threading import Thread	16	from threading import Thread
@@ -17,12 +20,39 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
17	from schainpy.model.data.jrodata import JROData	20	from schainpy.model.data.jrodata import JROData
18	from schainpy.utils import log	21	from schainpy.utils import log
19		22
20	MAXNUMX = 100	23	MAXNUMX = 500
21	MAXNUMY = 100	24	MAXNUMY = 500
22		25
23	class PrettyFloat(float):	26	PLOT_CODES = {
24	def __repr__(self):	27	'rti': 0, # Range time intensity (RTI).
25	return '%.2f' % self	28	'spc': 1, # Spectra (and Cross-spectra) information.
		29	'cspc': 2, # Cross-Correlation information.
		30	'coh': 3, # Coherence map.
		31	'base': 4, # Base lines graphic.
		32	'row': 5, # Row Spectra.
		33	'total': 6, # Total Power.
		34	'drift': 7, # Drifts graphics.
		35	'height': 8, # Height profile.
		36	'phase': 9, # Signal Phase.
		37	'power': 16,
		38	'noise': 17,
		39	'beacon': 18,
		40	'wind': 22,
		41	'skymap': 23,
		42	'Unknown': 24,
		43	'V-E': 25, # PIP Velocity.
		44	'Z-E': 26, # PIP Reflectivity.
		45	'V-A': 27, # RHI Velocity.
		46	'Z-A': 28, # RHI Reflectivity.
		47	}
		48
		49	def get_plot_code(s):
		50	label = s.split('_')[0]
		51	codes = [key for key in PLOT_CODES if key in label]
		52	if codes:
		53	return PLOT_CODES[codes[0]]
		54	else:
		55	return 24
26		56
27	def roundFloats(obj):	57	def roundFloats(obj):
28	if isinstance(obj, list):	58	if isinstance(obj, list):
@@ -82,12 +112,14 class Data(object):
82	Object to hold data to be plotted	112	Object to hold data to be plotted
83	'''	113	'''
84		114
85	def __init__(self, plottypes, throttle_value, exp_code):	115	def __init__(self, plottypes, throttle_value, exp_code, buffering=True):
86	self.plottypes = plottypes	116	self.plottypes = plottypes
87	self.throttle = throttle_value	117	self.throttle = throttle_value
88	self.exp_code = exp_code	118	self.exp_code = exp_code
		119	self.buffering = buffering
89	self.ended = False	120	self.ended = False
90	self.localtime = False	121	self.localtime = False
		122	self.meta = {}
91	self.__times = []	123	self.__times = []
92	self.__heights = []	124	self.__heights = []
93		125
@@ -102,7 +134,7 class Data(object):
102	if key not in self.data:	134	if key not in self.data:
103	raise KeyError(log.error('Missing key: {}'.format(key)))	135	raise KeyError(log.error('Missing key: {}'.format(key)))
104		136
105	if 'spc' in key:	137	if 'spc' in key or not self.buffering:
106	ret = self.data[key]	138	ret = self.data[key]
107	else:	139	else:
108	ret = numpy.array([self.data[key][x] for x in self.times])	140	ret = numpy.array([self.data[key][x] for x in self.times])
@@ -118,6 +150,7 class Data(object):
118	Configure object	150	Configure object
119	'''	151	'''
120		152
		153	self.type = ''
121	self.ended = False	154	self.ended = False
122	self.data = {}	155	self.data = {}
123	self.__times = []	156	self.__times = []
@@ -134,7 +167,7 class Data(object):
134	'''	167	'''
135		168
136	if len(self.data[key]):	169	if len(self.data[key]):
137	if 'spc' in key:	170	if 'spc' in key or not self.buffering:
138	return self.data[key].shape	171	return self.data[key].shape
139	return self.data[key][self.__times[0]].shape	172	return self.data[key][self.__times[0]].shape
140	return (0,)	173	return (0,)
@@ -147,9 +180,12 class Data(object):
147	if tm in self.__times:	180	if tm in self.__times:
148	return	181	return
149		182
		183	self.type = dataOut.type
150	self.parameters = getattr(dataOut, 'parameters', [])	184	self.parameters = getattr(dataOut, 'parameters', [])
151	if hasattr(dataOut, 'pairsList'):	185	if hasattr(dataOut, 'pairsList'):
152	self.pairs = dataOut.pairsList	186	self.pairs = dataOut.pairsList
		187	if hasattr(dataOut, 'meta'):
		188	self.meta = dataOut.meta
153	self.channels = dataOut.channelList	189	self.channels = dataOut.channelList
154	self.interval = dataOut.getTimeInterval()	190	self.interval = dataOut.getTimeInterval()
155	self.localtime = dataOut.useLocalTime	191	self.localtime = dataOut.useLocalTime
@@ -162,31 +198,39 class Data(object):
162	for plot in self.plottypes:	198	for plot in self.plottypes:
163	if plot == 'spc':	199	if plot == 'spc':
164	z = dataOut.data_spc/dataOut.normFactor	200	z = dataOut.data_spc/dataOut.normFactor
165	~~self~~.~~data~~[~~plot~~] = 10*numpy.log10(z)	201	buffer = 10*numpy.log10(z)
166	if plot == 'cspc':	202	if plot == 'cspc':
167	~~self~~.~~data~~[~~plot~~] = dataOut.data_cspc	203	buffer = dataOut.data_cspc
168	if plot == 'noise':	204	if plot == 'noise':
169	~~self~~.~~data~~[~~plot~~][tm] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)	205	buffer = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
170	if plot == 'rti':	206	if plot == 'rti':
171	~~self~~.~~data~~[~~plot~~][tm] = dataOut.getPower()	207	buffer = dataOut.getPower()
172	if plot == 'snr_db':	208	if plot == 'snr_db':
173	~~self~~.~~data~~[~~'snr'~~][tm] = dataOut.data_SNR	209	buffer = dataOut.data_SNR
174	if plot == 'snr':	210	if plot == 'snr':
175	~~self~~.~~data~~[~~plot~~][tm] = 10*numpy.log10(dataOut.data_SNR)	211	buffer = 10*numpy.log10(dataOut.data_SNR)
176	if plot == 'dop':	212	if plot == 'dop':
177	~~self~~.~~data~~[~~plot~~][tm] = 10*numpy.log10(dataOut.data_DOP)	213	buffer = 10*numpy.log10(dataOut.data_DOP)
178	if plot == 'mean':	214	if plot == 'mean':
179	~~self~~.~~data~~[~~plot~~][tm] = dataOut.data_MEAN	215	buffer = dataOut.data_MEAN
180	if plot == 'std':	216	if plot == 'std':
181	~~self~~.~~data~~[~~plot~~][tm] = dataOut.data_STD	217	buffer = dataOut.data_STD
182	if plot == 'coh':	218	if plot == 'coh':
183	~~self~~.~~data~~[~~plot~~][tm] = dataOut.getCoherence()	219	buffer = dataOut.getCoherence()
184	if plot == 'phase':	220	if plot == 'phase':
185	~~self~~.~~data~~[~~plot~~][tm] = dataOut.getCoherence(phase=True)	221	buffer = dataOut.getCoherence(phase=True)
186	if plot == 'output':	222	if plot == 'output':
187	~~self~~.~~data~~[~~plot~~][tm] = dataOut.data_output	223	buffer = dataOut.data_output
188	if plot == 'param':	224	if plot == 'param':
189	~~self~~.~~data~~[~~plot~~][tm] = dataOut.data_param	225	buffer = dataOut.data_param
		226
		227	if 'spc' in plot:
		228	self.data[plot] = buffer
		229	else:
		230	if self.buffering:
		231	self.data[plot][tm] = buffer
		232	else:
		233	self.data[plot] = buffer
190		234
191	def normalize_heights(self):	235	def normalize_heights(self):
192	'''	236	'''
@@ -220,7 +264,7 class Data(object):
220	tm = self.times[-1]	264	tm = self.times[-1]
221	dy = int(self.heights.size/MAXNUMY) + 1	265	dy = int(self.heights.size/MAXNUMY) + 1
222	for key in self.data:	266	for key in self.data:
223	if key in ('spc', 'cspc'):	267	if key in ('spc', 'cspc') or not self.buffering:
224	dx = int(self.data[key].shape[1]/MAXNUMX) + 1	268	dx = int(self.data[key].shape[1]/MAXNUMX) + 1
225	data[key] = roundFloats(self.data[key][::, ::dx, ::dy].tolist())	269	data[key] = roundFloats(self.data[key][::, ::dx, ::dy].tolist())
226	else:	270	else:
@@ -240,6 +284,10 class Data(object):
240	ret['pairs'] = self.pairs	284	ret['pairs'] = self.pairs
241	else:	285	else:
242	ret['pairs'] = []	286	ret['pairs'] = []
		287
		288	for key, value in self.meta.items():
		289	ret[key] = value
		290
243	return json.dumps(ret)	291	return json.dumps(ret)
244		292
245	@property	293	@property
@@ -492,7 +540,7 class PlotterReceiver(ProcessingUnit, Process):
492		540
493	throttle_value = 5	541	throttle_value = 5
494	__attrs__ = ['server', 'plottypes', 'realtime', 'localtime', 'throttle',	542	__attrs__ = ['server', 'plottypes', 'realtime', 'localtime', 'throttle',
495	'exp_code', 'web_server']	543	'exp_code', 'web_server', 'buffering']
496		544
497	def __init__(self, **kwargs):	545	def __init__(self, **kwargs):
498		546
@@ -513,6 +561,7 class PlotterReceiver(ProcessingUnit, Process):
513	self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')]	561	self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')]
514	self.realtime = kwargs.get('realtime', False)	562	self.realtime = kwargs.get('realtime', False)
515	self.localtime = kwargs.get('localtime', True)	563	self.localtime = kwargs.get('localtime', True)
		564	self.buffering = kwargs.get('buffering', True)
516	self.throttle_value = kwargs.get('throttle', 5)	565	self.throttle_value = kwargs.get('throttle', 5)
517	self.exp_code = kwargs.get('exp_code', None)	566	self.exp_code = kwargs.get('exp_code', None)
518	self.sendData = self.initThrottle(self.throttle_value)	567	self.sendData = self.initThrottle(self.throttle_value)
@@ -521,8 +570,8 class PlotterReceiver(ProcessingUnit, Process):
521		570
522	def setup(self):	571	def setup(self):
523		572
524	self.data = Data(self.plottypes, self.throttle_value, self.exp_code)	573	self.data = Data(self.plottypes, self.throttle_value, self.exp_code, self.buffering)
525	self.isConfig = True	574	self.isConfig = True
526		575
527	def event_monitor(self, monitor):	576	def event_monitor(self, monitor):
528		577
@@ -556,12 +605,12 class PlotterReceiver(ProcessingUnit, Process):
556	return sendDataThrottled	605	return sendDataThrottled
557		606
558	def send(self, data):	607	def send(self, data):
559	log.~~success~~('Sending {}'.format(data), self.name)	608	log.log('Sending {}'.format(data), self.name)
560	self.sender.send_pyobj(data)	609	self.sender.send_pyobj(data)
561		610
562	def run(self):	611	def run(self):
563		612
564	log.~~success~~(	613	log.log(
565	'Starting from {}'.format(self.address),	614	'Starting from {}'.format(self.address),
566	self.name	615	self.name
567	)	616	)
@@ -659,3 +708,157 class PlotterReceiver(ProcessingUnit, Process):
659	coerce = False	708	coerce = False
660		709
661	return	710	return
		711
		712
		713	class SendToFTP(Operation, Process):
		714
		715	'''
		716	Operation to send data over FTP.
		717	'''
		718
		719	__attrs__ = ['server', 'username', 'password', 'patterns', 'timeout']
		720
		721	def __init__(self, **kwargs):
		722	'''
		723	patterns = [(local1, remote1, ext, delay, exp_code, sub_exp_code), ...]
		724	'''
		725	Operation.__init__(self, **kwargs)
		726	Process.__init__(self)
		727	self.server = kwargs.get('server')
		728	self.username = kwargs.get('username')
		729	self.password = kwargs.get('password')
		730	self.patterns = kwargs.get('patterns')
		731	self.timeout = kwargs.get('timeout', 30)
		732	self.times = [time.time() for p in self.patterns]
		733	self.latest = ['' for p in self.patterns]
		734	self.mp = False
		735	self.ftp = None
		736
		737	def setup(self):
		738
		739	log.log('Connecting to ftp://{}'.format(self.server), self.name)
		740	try:
		741	self.ftp = ftplib.FTP(self.server, timeout=self.timeout)
		742	except ftplib.all_errors:
		743	log.error('Server connection fail: {}'.format(self.server), self.name)
		744	if self.ftp is not None:
		745	self.ftp.close()
		746	self.ftp = None
		747	self.isConfig = False
		748	return
		749
		750	try:
		751	self.ftp.login(self.username, self.password)
		752	except ftplib.all_errors:
		753	log.error('The given username y/o password are incorrect', self.name)
		754	if self.ftp is not None:
		755	self.ftp.close()
		756	self.ftp = None
		757	self.isConfig = False
		758	return
		759
		760	log.success('Connection success', self.name)
		761	self.isConfig = True
		762	return
		763
		764	def check(self):
		765
		766	try:
		767	self.ftp.voidcmd("NOOP")
		768	except:
		769	log.warning('Connection lost... trying to reconnect', self.name)
		770	if self.ftp is not None:
		771	self.ftp.close()
		772	self.ftp = None
		773	self.setup()
		774
		775	def find_files(self, path, ext):
		776
		777	files = glob.glob1(path, '*{}'.format(ext))
		778	files.sort()
		779	if files:
		780	return files[-1]
		781	return None
		782
		783	def getftpname(self, filename, exp_code, sub_exp_code):
		784
		785	thisDatetime = datetime.datetime.strptime(filename.split('_')[1], '%Y%m%d')
		786	YEAR_STR = '%4.4d'%thisDatetime.timetuple().tm_year
		787	DOY_STR = '%3.3d'%thisDatetime.timetuple().tm_yday
		788	exp_code = '%3.3d'%exp_code
		789	sub_exp_code = '%2.2d'%sub_exp_code
		790	plot_code = '%2.2d'% get_plot_code(filename)
		791	name = YEAR_STR + DOY_STR + '00' + exp_code + sub_exp_code + plot_code + '00.png'
		792	return name
		793
		794	def upload(self, src, dst):
		795
		796	log.log('Uploading {} '.format(src), self.name, nl=False)
		797
		798	fp = open(src, 'rb')
		799	command = 'STOR {}'.format(dst)
		800
		801	try:
		802	self.ftp.storbinary(command, fp, blocksize=1024)
		803	except Exception, e:
		804	log.error('{}'.format(e), self.name)
		805	if self.ftp is not None:
		806	self.ftp.close()
		807	self.ftp = None
		808	return 0
		809
		810	try:
		811	self.ftp.sendcmd('SITE CHMOD 755 {}'.format(dst))
		812	except Exception, e:
		813	log.error('{}'.format(e), self.name)
		814	if self.ftp is not None:
		815	self.ftp.close()
		816	self.ftp = None
		817	return 0
		818
		819	fp.close()
		820	log.success('OK', tag='')
		821	return 1
		822
		823	def send_files(self):
		824
		825	for x, pattern in enumerate(self.patterns):
		826	local, remote, ext, delay, exp_code, sub_exp_code = pattern
		827	if time.time()-self.times[x] >= delay:
		828	srcname = self.find_files(local, ext)
		829	src = os.path.join(local, srcname)
		830	if os.path.getmtime(src) < time.time() - 30*60:
		831	continue
		832
		833	if srcname is None or srcname == self.latest[x]:
		834	continue
		835
		836	if 'png' in ext:
		837	dstname = self.getftpname(srcname, exp_code, sub_exp_code)
		838	else:
		839	dstname = srcname
		840
		841	dst = os.path.join(remote, dstname)
		842
		843	if self.upload(src, dst):
		844	self.times[x] = time.time()
		845	self.latest[x] = srcname
		846	else:
		847	self.isConfig = False
		848	break
		849
		850	def run(self):
		851
		852	while True:
		853	if not self.isConfig:
		854	self.setup()
		855	if self.ftp is not None:
		856	self.check()
		857	self.send_files()
		858	time.sleep(10)
		859
		860	def close():
		861
		862	if self.ftp is not None:
		863	self.ftp.close()
		864	self.terminate()

schainpy/utils/log.py +20 -8

             import click
-            def warning(message, tag='Warning'):
+            def warning(message, tag='Warning', nl=True):
-                click.echo(click.style('[{}] {}'.format(tag, message), fg='yellow'))
+                if tag:
+                    click.echo(click.style('[{}] {}'.format(tag, message), fg='yellow'), nl=nl)
+                else:
+                    click.echo(click.style('{}'.format(message), fg='yellow'), nl=nl)
                 pass
-            def error(message, tag='Error'):
+            def error(message, tag='Error', nl=True):
-                click.echo(click.style('[{}] {}'.format(tag, message), fg='red'))
+                if tag:
+                    click.echo(click.style('[{}] {}'.format(tag, message), fg='red'), nl=nl)
+                else:
+                    click.echo(click.style('{}'.format(message), fg='red'), nl=nl)
                 pass
-            def success(message, tag='Info'):
+            def success(message, tag='Success', nl=True):
-                click.echo(click.style('[{}] {}'.format(tag, message), fg='green'))
+                if tag:
+                    click.echo(click.style('[{}] {}'.format(tag, message), fg='green'), nl=nl)
+                else:
+                    click.echo(click.style('{}'.format(message), fg='green'), nl=nl)
                 pass
-            def log(message, tag='Info'):
+            def log(message, tag='Info', nl=True):
-                click.echo('[{}] {}'.format(tag, message))
+                if tag:
+                    click.echo('[{}] {}'.format(tag, message), nl=nl)
+                else:
+                    click.echo('{}'.format(message), nl=nl)
                 pass

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages