##// END OF EJS Templates
schain-jc
RSS
Merge with px1000 branch
Juan C. Espinoza -
r1164:80b8ab8c0c11 merge
parent child
Show More
Side by Side Unified
Context file:
r1164:80b8ab8c0c11 Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,350
1 '''
2 Created on Jan 15, 2018
3
4 @author: Juan C. Espinoza
5 '''
6
7 import os
8 import sys
9 import time
10 import glob
11 import datetime
12 import tarfile
13
14 import numpy
15 try:
16 from netCDF4 import Dataset
17 except:
18 log.warning(
19 'You should install "netCDF4" module if you want to read/write NCDF files'
20 )
21
22 from utils import folder_in_range
23
24 from schainpy.model.io.jroIO_base import JRODataReader
25 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
26 from schainpy.model.data.jrodata import Parameters
27 from schainpy.utils import log
28
29 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
30
31
32 class PXReader(JRODataReader, ProcessingUnit):
33
34 def __init__(self, **kwargs):
35
36 ProcessingUnit.__init__(self, **kwargs)
37
38 self.dataOut = Parameters()
39 self.counter_records = 0
40 self.nrecords = None
41 self.flagNoMoreFiles = 0
42 self.isConfig = False
43 self.filename = None
44 self.intervals = set()
45 self.ext = ('.nc', '.tgz')
46 self.online_mode = False
47
48 def setup(self,
49 path=None,
50 startDate=None,
51 endDate=None,
52 format=None,
53 startTime=datetime.time(0, 0, 0),
54 endTime=datetime.time(23, 59, 59),
55 walk=False,
56 **kwargs):
57
58 self.path = path
59 self.startDate = startDate
60 self.endDate = endDate
61 self.startTime = startTime
62 self.endTime = endTime
63 self.datatime = datetime.datetime(1900,1,1)
64 self.walk = walk
65 self.nTries = kwargs.get('nTries', 10)
66 self.online = kwargs.get('online', False)
67 self.delay = kwargs.get('delay', 60)
68 self.ele = kwargs.get('ext', '')
69
70 if self.path is None:
71 raise ValueError, 'The path is not valid'
72
73 self.search_files(path, startDate, endDate, startTime, endTime, walk)
74 self.cursor = 0
75 self.counter_records = 0
76
77 if not self.files:
78 raise Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path)
79
80 def search_files(self, path, startDate, endDate, startTime, endTime, walk):
81 '''
82 Searching for NCDF files in path
83 Creating a list of files to procces included in [startDate,endDate]
84
85 Input:
86 path - Path to find files
87 '''
88
89 log.log('Searching files {} in {} '.format(self.ext, path), 'PXReader')
90 if walk:
91 paths = [os.path.join(path, p) for p in os.listdir(path) if os.path.isdir(os.path.join(path, p))]
92 paths.sort()
93 else:
94 paths = [path]
95
96 fileList0 = []
97
98 for subpath in paths:
99 if not folder_in_range(subpath.split('/')[-1], startDate, endDate, '%Y%m%d'):
100 continue
101 fileList0 += [os.path.join(subpath, s) for s in glob.glob1(subpath, '*') if os.path.splitext(s)[-1] in self.ext and '{}'.format(self.ele) in s]
102
103 fileList0.sort()
104 if self.online:
105 fileList0 = fileList0[-1:]
106
107 self.files = {}
108
109 startDate = startDate - datetime.timedelta(1)
110 endDate = endDate + datetime.timedelta(1)
111
112 for fullname in fileList0:
113 thisFile = fullname.split('/')[-1]
114 year = thisFile[3:7]
115 if not year.isdigit():
116 continue
117
118 month = thisFile[7:9]
119 if not month.isdigit():
120 continue
121
122 day = thisFile[9:11]
123 if not day.isdigit():
124 continue
125
126 year, month, day = int(year), int(month), int(day)
127 dateFile = datetime.date(year, month, day)
128 timeFile = datetime.time(int(thisFile[12:14]), int(thisFile[14:16]), int(thisFile[16:18]))
129
130 if (startDate > dateFile) or (endDate < dateFile):
131 continue
132
133 dt = datetime.datetime.combine(dateFile, timeFile)
134 if dt not in self.files:
135 self.files[dt] = []
136 self.files[dt].append(fullname)
137
138 self.dates = self.files.keys()
139 self.dates.sort()
140
141 return
142
143 def search_files_online(self):
144 '''
145 Searching for NCDF files in online mode path
146 Creating a list of files to procces included in [startDate,endDate]
147
148 Input:
149 path - Path to find files
150 '''
151
152 self.files = {}
153
154 for n in range(self.nTries):
155
156 if self.walk:
157 paths = [os.path.join(self.path, p) for p in os.listdir(self.path) if os.path.isdir(os.path.join(self.path, p))]
158 paths.sort()
159 path = paths[-1]
160 else:
161 path = self.path
162
163 new_files = [os.path.join(path, s) for s in glob.glob1(path, '*') if os.path.splitext(s)[-1] in self.ext and '{}'.format(self.ele) in s]
164 new_files.sort()
165
166 for fullname in new_files:
167 thisFile = fullname.split('/')[-1]
168 year = thisFile[3:7]
169 if not year.isdigit():
170 continue
171
172 month = thisFile[7:9]
173 if not month.isdigit():
174 continue
175
176 day = thisFile[9:11]
177 if not day.isdigit():
178 continue
179
180 year, month, day = int(year), int(month), int(day)
181 dateFile = datetime.date(year, month, day)
182 timeFile = datetime.time(int(thisFile[12:14]), int(thisFile[14:16]), int(thisFile[16:18]))
183
184 dt = datetime.datetime.combine(dateFile, timeFile)
185
186 if self.dt >= dt:
187 continue
188
189 if dt not in self.files:
190 self.dt = dt
191 self.files[dt] = []
192
193 self.files[dt].append(fullname)
194 break
195
196 if self.files:
197 break
198 else:
199 log.warning('Waiting {} seconds for the next file, try {} ...'.format(self.delay, n + 1), 'PXReader')
200 time.sleep(self.delay)
201
202 if not self.files:
203 return 0
204
205 self.dates = self.files.keys()
206 self.dates.sort()
207 self.cursor = 0
208
209 return 1
210
211 def parseFile(self):
212 '''
213 '''
214
215 header = {}
216
217 for attr in self.fp.ncattrs():
218 header[str(attr)] = getattr(self.fp, attr)
219
220 self.header.append(header)
221
222 self.data[header['TypeName']] = numpy.array(self.fp.variables[header['TypeName']])
223
224 def setNextFile(self):
225 '''
226 Open next files for the current datetime
227 '''
228
229 cursor = self.cursor
230 if not self.online_mode:
231 if cursor == len(self.dates):
232 if self.online:
233 cursor = 0
234 self.dt = self.dates[cursor]
235 self.online_mode = True
236 if not self.search_files_online():
237 log.success('No more files', 'PXReader')
238 return 0
239 else:
240 log.success('No more files', 'PXReader')
241 self.flagNoMoreFiles = 1
242 return 0
243 else:
244 if not self.search_files_online():
245 return 0
246 cursor = self.cursor
247
248 self.data = {}
249 self.header = []
250
251 for fullname in self.files[self.dates[cursor]]:
252
253 log.log('Opening: {}'.format(fullname), 'PXReader')
254
255 if os.path.splitext(fullname)[-1] == '.tgz':
256 tar = tarfile.open(fullname, 'r:gz')
257 tar.extractall('/tmp')
258 files = [os.path.join('/tmp', member.name) for member in tar.getmembers()]
259 else:
260 files = [fullname]
261
262 for filename in files:
263 if self.filename is not None:
264 self.fp.close()
265
266 self.filename = filename
267 self.filedate = self.dates[cursor]
268 self.fp = Dataset(self.filename, 'r')
269 self.parseFile()
270
271 self.counter_records += 1
272 self.cursor += 1
273 return 1
274
275 def readNextFile(self):
276
277 while True:
278 self.flagDiscontinuousBlock = 0
279 if not self.setNextFile():
280 return 0
281
282 self.datatime = datetime.datetime.utcfromtimestamp(self.header[0]['Time'])
283
284 if self.online:
285 break
286
287 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
288 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
289 log.warning(
290 'Reading Record No. {}/{} -> {} [Skipping]'.format(
291 self.counter_records,
292 self.nrecords,
293 self.datatime.ctime()),
294 'PXReader')
295 continue
296 break
297
298 log.log(
299 'Reading Record No. {}/{} -> {}'.format(
300 self.counter_records,
301 self.nrecords,
302 self.datatime.ctime()),
303 'PXReader')
304
305 return 1
306
307
308 def set_output(self):
309 '''
310 Storing data from buffer to dataOut object
311 '''
312
313 self.data['Elevation'] = numpy.array(self.fp.variables['Elevation'])
314 self.data['Azimuth'] = numpy.array(self.fp.variables['Azimuth'])
315 self.dataOut.range = numpy.array(self.fp.variables['GateWidth'])
316 self.dataOut.data = self.data
317 self.dataOut.units = [h['Unit-value'] for h in self.header]
318 self.dataOut.parameters = [h['TypeName'] for h in self.header]
319 self.dataOut.missing = self.header[0]['MissingData']
320 self.dataOut.max_range = self.header[0]['MaximumRange-value']
321 self.dataOut.elevation = self.header[0]['Elevation']
322 self.dataOut.azimuth = self.header[0]['Azimuth']
323 self.dataOut.latitude = self.header[0]['Latitude']
324 self.dataOut.longitude = self.header[0]['Longitude']
325 self.dataOut.utctime = self.header[0]['Time']
326 self.dataOut.utctimeInit = self.dataOut.utctime
327 self.dataOut.useLocalTime = True
328 self.dataOut.flagNoData = False
329 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
330
331 log.log('Parameters found: {}'.format(','.join(self.dataOut.parameters)),
332 'PXReader')
333
334 def getData(self):
335 '''
336 Storing data from databuffer to dataOut object
337 '''
338 if self.flagNoMoreFiles:
339 self.dataOut.flagNoData = True
340 log.error('No file left to process', 'PXReader')
341 return 0
342
343 if not self.readNextFile():
344 self.dataOut.flagNoData = True
345 return 0
346
347 self.set_output()
348
349 return 1
350
Show file before | Show file after | Hide comments
@@ -0,0 +1,24
1 """
2 Utilities for IO modules
3 """
4
5 import os
6 from datetime import datetime
7
8 def folder_in_range(folder, start_date, end_date, pattern):
9 """
10 Check whether folder is bettwen start_date and end_date
11
12 Args:
13 folder (str): Folder to check
14 start_date (date): Initial date
15 end_date (date): Final date
16 pattern (str): Datetime format of the folder
17 Returns:
18 bool: True for success, False otherwise
19 """
20 try:
21 dt = datetime.strptime(folder, pattern)
22 except:
23 raise ValueError('Folder {} does not match {} format'.format(folder, pattern))
24 return start_date <= dt.date() <= end_date
Show file before | Show file after | Hide comments
@@ -0,0 +1,64
1 '''
2 Created on Oct 24, 2016
3
4 @author: roj- LouVD
5 '''
6
7 import numpy
8 import datetime
9 import time
10 from time import gmtime
11
12 from numpy import transpose
13
14 from jroproc_base import ProcessingUnit, Operation
15 from schainpy.model.data.jrodata import Parameters
16
17
18 class PXParametersProc(ProcessingUnit):
19 '''
20 Processing unit for PX parameters data
21 '''
22
23 def __init__(self, **kwargs):
24 """
25 Inputs: None
26 """
27 ProcessingUnit.__init__(self, **kwargs)
28 self.dataOut = Parameters()
29 self.isConfig = False
30
31 def setup(self, mode):
32 """
33 """
34 self.dataOut.mode = mode
35
36 def run(self, mode):
37 """
38 Args:
39 mode (str): select independent variable 'E' for elevation or 'A' for azimuth
40 """
41
42 if not self.isConfig:
43 self.setup(mode)
44 self.isConfig = True
45
46 if self.dataIn.type == 'Parameters':
47 self.dataOut.copy(self.dataIn)
48
49 self.dataOut.data_param = numpy.array([self.dataOut.data[var] for var in self.dataOut.parameters])
50 self.dataOut.data_param[self.dataOut.data_param == self.dataOut.missing] = numpy.nan
51
52 if mode.upper()=='E':
53 self.dataOut.heightList = self.dataOut.data['Azimuth']
54 else:
55 self.dataOut.heightList = self.dataOut.data['Elevation']
56
57 attrs = ['units', 'elevation', 'azimuth', 'max_range', 'latitude', 'longitude']
58 meta = {}
59
60 for attr in attrs:
61 meta[attr] = getattr(self.dataOut, attr)
62
63 meta['mode'] = mode
64 self.dataOut.meta = meta No newline at end of file
Show file before | Show file after | Hide comments
@@ -1282,7 +1282,7 class Project(Process):
1282 1282
1283 1283 def run(self):
1284 1284
1285 log.success('Starting {}'.format(self.name))
1285 log.success('Starting {}'.format(self.name), tag='')
1286 1286 self.start_time = time.time()
1287 1287 self.createObjects()
1288 1288 self.connectObjects()
Show file before | Show file after | Hide comments
@@ -9,6 +9,7 import zmq
9 9 import numpy
10 10 import matplotlib
11 11 import matplotlib.pyplot as plt
12 from matplotlib.patches import Polygon
12 13 from mpl_toolkits.axes_grid1 import make_axes_locatable
13 14 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
14 15
@@ -24,6 +25,23 matplotlib.pyplot.register_cmap(cmap=ncmap)
24 25
25 26 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis', 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
26 27
28 EARTH_RADIUS = 6.3710e3
29
30 def ll2xy(lat1, lon1, lat2, lon2):
31
32 p = 0.017453292519943295
33 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
34 r = 12742 * numpy.arcsin(numpy.sqrt(a))
35 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)*numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
36 theta = -theta + numpy.pi/2
37 return r*numpy.cos(theta), r*numpy.sin(theta)
38
39 def km2deg(km):
40 '''
41 Convert distance in km to degrees
42 '''
43
44 return numpy.rad2deg(km/EARTH_RADIUS)
27 45
28 46 def figpause(interval):
29 47 backend = plt.rcParams['backend']
@@ -64,7 +82,7 class PlotData(Operation, Process):
64 82 __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
65 83 'zlimits', 'xlabel', 'ylabel', 'xaxis','cb_label', 'title',
66 84 'colorbar', 'bgcolor', 'width', 'height', 'localtime', 'oneFigure',
67 'showprofile', 'decimation']
85 'showprofile', 'decimation', 'ftp']
68 86
69 87 def __init__(self, **kwargs):
70 88
@@ -81,6 +99,7 class PlotData(Operation, Process):
81 99 self.localtime = kwargs.pop('localtime', True)
82 100 self.show = kwargs.get('show', True)
83 101 self.save = kwargs.get('save', False)
102 self.ftp = kwargs.get('ftp', False)
84 103 self.colormap = kwargs.get('colormap', self.colormap)
85 104 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
86 105 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
@@ -90,6 +109,7 class PlotData(Operation, Process):
90 109 self.title = kwargs.get('wintitle', self.CODE.upper())
91 110 self.cb_label = kwargs.get('cb_label', None)
92 111 self.cb_labels = kwargs.get('cb_labels', None)
112 self.labels = kwargs.get('labels', None)
93 113 self.xaxis = kwargs.get('xaxis', 'frequency')
94 114 self.zmin = kwargs.get('zmin', None)
95 115 self.zmax = kwargs.get('zmax', None)
@@ -97,8 +117,10 class PlotData(Operation, Process):
97 117 self.xmin = kwargs.get('xmin', None)
98 118 self.xmax = kwargs.get('xmax', None)
99 119 self.xrange = kwargs.get('xrange', 24)
120 self.xscale = kwargs.get('xscale', None)
100 121 self.ymin = kwargs.get('ymin', None)
101 122 self.ymax = kwargs.get('ymax', None)
123 self.yscale = kwargs.get('yscale', None)
102 124 self.xlabel = kwargs.get('xlabel', None)
103 125 self.decimation = kwargs.get('decimation', None)
104 126 self.showSNR = kwargs.get('showSNR', False)
@@ -107,8 +129,10 class PlotData(Operation, Process):
107 129 self.height = kwargs.get('height', None)
108 130 self.colorbar = kwargs.get('colorbar', True)
109 131 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
132 self.channels = kwargs.get('channels', None)
110 133 self.titles = kwargs.get('titles', [])
111 134 self.polar = False
135 self.grid = kwargs.get('grid', False)
112 136
113 137 def __fmtTime(self, x, pos):
114 138 '''
@@ -381,14 +405,19 class PlotData(Operation, Process):
381 405 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
382 406 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
383 407
384 Y = numpy.array([5, 10, 20, 50, 100, 200, 500, 1000, 2000])
385 i = 1 if numpy.where(ymax-ymin < Y)[0][0] < 0 else numpy.where(ymax-ymin < Y)[0][0]
386 ystep = Y[i] / 5
408 Y = numpy.array([1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000])
409 i = 1 if numpy.where(abs(ymax-ymin) <= Y)[0][0] < 0 else numpy.where(abs(ymax-ymin) <= Y)[0][0]
410 ystep = Y[i] / 10.
387 411
388 412 for n, ax in enumerate(self.axes):
389 413 if ax.firsttime:
390 414 ax.set_facecolor(self.bgcolor)
391 415 ax.yaxis.set_major_locator(MultipleLocator(ystep))
416 ax.xaxis.set_major_locator(MultipleLocator(ystep))
417 if self.xscale:
418 ax.xaxis.set_major_formatter(FuncFormatter(lambda x, pos: '{0:g}'.format(x*self.xscale)))
419 if self.xscale:
420 ax.yaxis.set_major_formatter(FuncFormatter(lambda x, pos: '{0:g}'.format(x*self.yscale)))
392 421 if self.xaxis is 'time':
393 422 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
394 423 ax.xaxis.set_major_locator(LinearLocator(9))
@@ -414,13 +443,15 class PlotData(Operation, Process):
414 443 ax.cbar.set_label(self.cb_labels[n], size=8)
415 444 else:
416 445 ax.cbar = None
446 if self.grid:
447 ax.grid(True)
417 448
418 449 if not self.polar:
419 450 ax.set_xlim(xmin, xmax)
420 451 ax.set_ylim(ymin, ymax)
421 ax.set_title('{} - {} {}'.format(
452 ax.set_title('{} {} {}'.format(
422 453 self.titles[n],
423 self.getDateTime(self.max_time).strftime('%H:%M:%S'),
454 self.getDateTime(self.max_time).strftime('%Y-%m-%dT%H:%M:%S'),
424 455 self.time_label),
425 456 size=8)
426 457 else:
@@ -432,13 +463,15 class PlotData(Operation, Process):
432 463 def __plot(self):
433 464 '''
434 465 '''
435 log.success('Plotting', self.name)
466 log.log('Plotting', self.name)
436 467
437 468 try:
438 469 self.plot()
439 470 self.format()
440 except:
471 except Exception as e:
441 472 log.warning('{} Plot could not be updated... check data'.format(self.CODE), self.name)
473 log.error(str(e), '')
474 return
442 475
443 476 for n, fig in enumerate(self.figures):
444 477 if self.nrows == 0 or self.nplots == 0:
@@ -452,14 +485,20 class PlotData(Operation, Process):
452 485 self.getDateTime(self.max_time).strftime('%Y/%m/%d')))
453 486 fig.canvas.draw()
454 487
455 if self.save and self.data.ended:
456 channels = range(self.nrows)
488 if self.save and (self.data.ended or not self.data.buffering):
489
490 if self.save_labels:
491 labels = self.save_labels
492 else:
493 labels = range(self.nrows)
494
457 495 if self.oneFigure:
458 496 label = ''
459 497 else:
460 label = '_{}'.format(channels[n])
498 label = '-{}'.format(labels[n])
461 499 figname = os.path.join(
462 500 self.save,
501 self.CODE,
463 502 '{}{}_{}.png'.format(
464 503 self.CODE,
465 504 label,
@@ -468,6 +507,8 class PlotData(Operation, Process):
468 507 )
469 508 )
470 509 log.log('Saving figure: {}'.format(figname), self.name)
510 if not os.path.isdir(os.path.dirname(figname)):
511 os.makedirs(os.path.dirname(figname))
471 512 fig.savefig(figname)
472 513
473 514 def plot(self):
@@ -477,7 +518,7 class PlotData(Operation, Process):
477 518
478 519 def run(self):
479 520
480 log.success('Starting', self.name)
521 log.log('Starting', self.name)
481 522
482 523 context = zmq.Context()
483 524 receiver = context.socket(zmq.SUB)
@@ -978,3 +1019,130 class PlotOutputData(PlotParamData):
978 1019
979 1020 CODE = 'output'
980 1021 colormap = 'seismic'
1022
1023
1024 class PlotPolarMapData(PlotData):
1025 '''
1026 Plot for meteors detection data
1027 '''
1028
1029 CODE = 'param'
1030 colormap = 'seismic'
1031
1032 def setup(self):
1033 self.ncols = 1
1034 self.nrows = 1
1035 self.width = 9
1036 self.height = 8
1037 self.mode = self.data.meta['mode']
1038 if self.channels is not None:
1039 self.nplots = len(self.channels)
1040 self.nrows = len(self.channels)
1041 else:
1042 self.nplots = self.data.shape(self.CODE)[0]
1043 self.nrows = self.nplots
1044 self.channels = range(self.nplots)
1045 if self.mode == 'E':
1046 self.xlabel = 'Longitude'
1047 self.ylabel = 'Latitude'
1048 else:
1049 self.xlabel = 'Range (km)'
1050 self.ylabel = 'Height (km)'
1051 self.bgcolor = 'white'
1052 self.cb_labels = self.data.meta['units']
1053 self.lat = self.data.meta['latitude']
1054 self.lon = self.data.meta['longitude']
1055 self.xmin, self.xmax = float(km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
1056 self.ymin, self.ymax = float(km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
1057 # self.polar = True
1058
1059 def plot(self):
1060
1061 for n, ax in enumerate(self.axes):
1062 data = self.data['param'][self.channels[n]]
1063
1064 zeniths = numpy.linspace(0, self.data.meta['max_range'], data.shape[1])
1065 if self.mode == 'E':
1066 azimuths = -numpy.radians(self.data.heights)+numpy.pi/2
1067 r, theta = numpy.meshgrid(zeniths, azimuths)
1068 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
1069 x = km2deg(x) + self.lon
1070 y = km2deg(y) + self.lat
1071 else:
1072 azimuths = numpy.radians(self.data.heights)
1073 r, theta = numpy.meshgrid(zeniths, azimuths)
1074 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
1075 self.y = zeniths
1076
1077 if ax.firsttime:
1078 if self.zlimits is not None:
1079 self.zmin, self.zmax = self.zlimits[n]
1080 ax.plt = ax.pcolormesh(#r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
1081 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
1082 vmin=self.zmin,
1083 vmax=self.zmax,
1084 cmap=self.cmaps[n])
1085 else:
1086 if self.zlimits is not None:
1087 self.zmin, self.zmax = self.zlimits[n]
1088 ax.collections.remove(ax.collections[0])
1089 ax.plt = ax.pcolormesh(# r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
1090 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
1091 vmin=self.zmin,
1092 vmax=self.zmax,
1093 cmap=self.cmaps[n])
1094
1095 if self.mode == 'A':
1096 continue
1097
1098 # plot district names
1099 f = open('/data/workspace/schain_scripts/distrito.csv')
1100 for line in f:
1101 label, lon, lat = [s.strip() for s in line.split(',') if s]
1102 lat = float(lat)
1103 lon = float(lon)
1104 # ax.plot(lon, lat, '.b', ms=2)
1105 ax.text(lon, lat, label.decode('utf8'), ha='center', va='bottom', size='8', color='black')
1106
1107 # plot limites
1108 limites =[]
1109 tmp = []
1110 for line in open('/data/workspace/schain_scripts/lima.csv'):
1111 if '#' in line:
1112 if tmp:
1113 limites.append(tmp)
1114 tmp = []
1115 continue
1116 values = line.strip().split(',')
1117 tmp.append((float(values[0]), float(values[1])))
1118 for points in limites:
1119 ax.add_patch(Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
1120
1121 # plot Cuencas
1122 for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
1123 f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
1124 values = [line.strip().split(',') for line in f]
1125 points = [(float(s[0]), float(s[1])) for s in values]
1126 ax.add_patch(Polygon(points, ec='b', fc='none'))
1127
1128 # plot grid
1129 for r in (15, 30, 45, 60):
1130 ax.add_artist(plt.Circle((self.lon, self.lat), km2deg(r), color='0.6', fill=False, lw=0.2))
1131 ax.text(
1132 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
1133 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
1134 '{}km'.format(r),
1135 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
1136
1137 if self.mode == 'E':
1138 title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
1139 label = 'E{:02d}'.format(int(self.data.meta['elevation']))
1140 else:
1141 title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
1142 label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
1143
1144 self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
1145 self.titles = ['{} {}'.format(self.data.parameters[x], title) for x in self.channels]
1146 self.saveTime = self.max_time
1147
1148
Show file before | Show file after | Hide comments
@@ -25,8 +25,8 class SpectraPlot(Figure):
25 25 self.isConfig = False
26 26 self.__nsubplots = 1
27 27
28 self.WIDTH = 300
29 self.HEIGHT = 300
28 self.WIDTH = 250
29 self.HEIGHT = 250
30 30 self.WIDTHPROF = 120
31 31 self.HEIGHTPROF = 0
32 32 self.counter_imagwr = 0
Show file before | Show file after | Hide comments
@@ -18,4 +18,6 from jroIO_madrigal import *
18 18 from bltrIO_param import *
19 19 from jroIO_bltr import *
20 20 from jroIO_mira35c import *
21 from julIO_param import * No newline at end of file
21 from julIO_param import *
22
23 from pxIO_param import * No newline at end of file
Show file before | Show file after | Hide comments
@@ -13,3 +13,4 from jroproc_parameters import *
13 13 from jroproc_spectra_lags import *
14 14 from jroproc_spectra_acf import *
15 15 from bltrproc_parameters import *
16 from pxproc_parameters import *
Show file before | Show file after | Hide comments
@@ -2,12 +2,15
2 2 @author: Juan C. Espinoza
3 3 '''
4 4
5 import os
6 import glob
5 7 import time
6 8 import json
7 9 import numpy
8 10 import paho.mqtt.client as mqtt
9 11 import zmq
10 12 import datetime
13 import ftplib
11 14 from zmq.utils.monitor import recv_monitor_message
12 15 from functools import wraps
13 16 from threading import Thread
@@ -17,12 +20,39 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
17 20 from schainpy.model.data.jrodata import JROData
18 21 from schainpy.utils import log
19 22
20 MAXNUMX = 100
21 MAXNUMY = 100
22
23 class PrettyFloat(float):
24 def __repr__(self):
25 return '%.2f' % self
23 MAXNUMX = 500
24 MAXNUMY = 500
25
26 PLOT_CODES = {
27 'rti': 0, # Range time intensity (RTI).
28 'spc': 1, # Spectra (and Cross-spectra) information.
29 'cspc': 2, # Cross-Correlation information.
30 'coh': 3, # Coherence map.
31 'base': 4, # Base lines graphic.
32 'row': 5, # Row Spectra.
33 'total': 6, # Total Power.
34 'drift': 7, # Drifts graphics.
35 'height': 8, # Height profile.
36 'phase': 9, # Signal Phase.
37 'power': 16,
38 'noise': 17,
39 'beacon': 18,
40 'wind': 22,
41 'skymap': 23,
42 'Unknown': 24,
43 'V-E': 25, # PIP Velocity.
44 'Z-E': 26, # PIP Reflectivity.
45 'V-A': 27, # RHI Velocity.
46 'Z-A': 28, # RHI Reflectivity.
47 }
48
49 def get_plot_code(s):
50 label = s.split('_')[0]
51 codes = [key for key in PLOT_CODES if key in label]
52 if codes:
53 return PLOT_CODES[codes[0]]
54 else:
55 return 24
26 56
27 57 def roundFloats(obj):
28 58 if isinstance(obj, list):
@@ -82,12 +112,14 class Data(object):
82 112 Object to hold data to be plotted
83 113 '''
84 114
85 def __init__(self, plottypes, throttle_value, exp_code):
115 def __init__(self, plottypes, throttle_value, exp_code, buffering=True):
86 116 self.plottypes = plottypes
87 117 self.throttle = throttle_value
88 118 self.exp_code = exp_code
119 self.buffering = buffering
89 120 self.ended = False
90 121 self.localtime = False
122 self.meta = {}
91 123 self.__times = []
92 124 self.__heights = []
93 125
@@ -102,7 +134,7 class Data(object):
102 134 if key not in self.data:
103 135 raise KeyError(log.error('Missing key: {}'.format(key)))
104 136
105 if 'spc' in key:
137 if 'spc' in key or not self.buffering:
106 138 ret = self.data[key]
107 139 else:
108 140 ret = numpy.array([self.data[key][x] for x in self.times])
@@ -118,6 +150,7 class Data(object):
118 150 Configure object
119 151 '''
120 152
153 self.type = ''
121 154 self.ended = False
122 155 self.data = {}
123 156 self.__times = []
@@ -134,7 +167,7 class Data(object):
134 167 '''
135 168
136 169 if len(self.data[key]):
137 if 'spc' in key:
170 if 'spc' in key or not self.buffering:
138 171 return self.data[key].shape
139 172 return self.data[key][self.__times[0]].shape
140 173 return (0,)
@@ -147,9 +180,12 class Data(object):
147 180 if tm in self.__times:
148 181 return
149 182
183 self.type = dataOut.type
150 184 self.parameters = getattr(dataOut, 'parameters', [])
151 185 if hasattr(dataOut, 'pairsList'):
152 186 self.pairs = dataOut.pairsList
187 if hasattr(dataOut, 'meta'):
188 self.meta = dataOut.meta
153 189 self.channels = dataOut.channelList
154 190 self.interval = dataOut.getTimeInterval()
155 191 self.localtime = dataOut.useLocalTime
@@ -162,31 +198,39 class Data(object):
162 198 for plot in self.plottypes:
163 199 if plot == 'spc':
164 200 z = dataOut.data_spc/dataOut.normFactor
165 self.data[plot] = 10*numpy.log10(z)
201 buffer = 10*numpy.log10(z)
166 202 if plot == 'cspc':
167 self.data[plot] = dataOut.data_cspc
203 buffer = dataOut.data_cspc
168 204 if plot == 'noise':
169 self.data[plot][tm] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
205 buffer = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
170 206 if plot == 'rti':
171 self.data[plot][tm] = dataOut.getPower()
207 buffer = dataOut.getPower()
172 208 if plot == 'snr_db':
173 self.data['snr'][tm] = dataOut.data_SNR
209 buffer = dataOut.data_SNR
174 210 if plot == 'snr':
175 self.data[plot][tm] = 10*numpy.log10(dataOut.data_SNR)
211 buffer = 10*numpy.log10(dataOut.data_SNR)
176 212 if plot == 'dop':
177 self.data[plot][tm] = 10*numpy.log10(dataOut.data_DOP)
213 buffer = 10*numpy.log10(dataOut.data_DOP)
178 214 if plot == 'mean':
179 self.data[plot][tm] = dataOut.data_MEAN
215 buffer = dataOut.data_MEAN
180 216 if plot == 'std':
181 self.data[plot][tm] = dataOut.data_STD
217 buffer = dataOut.data_STD
182 218 if plot == 'coh':
183 self.data[plot][tm] = dataOut.getCoherence()
219 buffer = dataOut.getCoherence()
184 220 if plot == 'phase':
185 self.data[plot][tm] = dataOut.getCoherence(phase=True)
221 buffer = dataOut.getCoherence(phase=True)
186 222 if plot == 'output':
187 self.data[plot][tm] = dataOut.data_output
223 buffer = dataOut.data_output
188 224 if plot == 'param':
189 self.data[plot][tm] = dataOut.data_param
225 buffer = dataOut.data_param
226
227 if 'spc' in plot:
228 self.data[plot] = buffer
229 else:
230 if self.buffering:
231 self.data[plot][tm] = buffer
232 else:
233 self.data[plot] = buffer
190 234
191 235 def normalize_heights(self):
192 236 '''
@@ -220,7 +264,7 class Data(object):
220 264 tm = self.times[-1]
221 265 dy = int(self.heights.size/MAXNUMY) + 1
222 266 for key in self.data:
223 if key in ('spc', 'cspc'):
267 if key in ('spc', 'cspc') or not self.buffering:
224 268 dx = int(self.data[key].shape[1]/MAXNUMX) + 1
225 269 data[key] = roundFloats(self.data[key][::, ::dx, ::dy].tolist())
226 270 else:
@@ -240,6 +284,10 class Data(object):
240 284 ret['pairs'] = self.pairs
241 285 else:
242 286 ret['pairs'] = []
287
288 for key, value in self.meta.items():
289 ret[key] = value
290
243 291 return json.dumps(ret)
244 292
245 293 @property
@@ -492,7 +540,7 class PlotterReceiver(ProcessingUnit, Process):
492 540
493 541 throttle_value = 5
494 542 __attrs__ = ['server', 'plottypes', 'realtime', 'localtime', 'throttle',
495 'exp_code', 'web_server']
543 'exp_code', 'web_server', 'buffering']
496 544
497 545 def __init__(self, **kwargs):
498 546
@@ -513,6 +561,7 class PlotterReceiver(ProcessingUnit, Process):
513 561 self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')]
514 562 self.realtime = kwargs.get('realtime', False)
515 563 self.localtime = kwargs.get('localtime', True)
564 self.buffering = kwargs.get('buffering', True)
516 565 self.throttle_value = kwargs.get('throttle', 5)
517 566 self.exp_code = kwargs.get('exp_code', None)
518 567 self.sendData = self.initThrottle(self.throttle_value)
@@ -521,8 +570,8 class PlotterReceiver(ProcessingUnit, Process):
521 570
522 571 def setup(self):
523 572
524 self.data = Data(self.plottypes, self.throttle_value, self.exp_code)
525 self.isConfig = True
573 self.data = Data(self.plottypes, self.throttle_value, self.exp_code, self.buffering)
574 self.isConfig = True
526 575
527 576 def event_monitor(self, monitor):
528 577
@@ -556,12 +605,12 class PlotterReceiver(ProcessingUnit, Process):
556 605 return sendDataThrottled
557 606
558 607 def send(self, data):
559 log.success('Sending {}'.format(data), self.name)
608 log.log('Sending {}'.format(data), self.name)
560 609 self.sender.send_pyobj(data)
561 610
562 611 def run(self):
563 612
564 log.success(
613 log.log(
565 614 'Starting from {}'.format(self.address),
566 615 self.name
567 616 )
@@ -659,3 +708,157 class PlotterReceiver(ProcessingUnit, Process):
659 708 coerce = False
660 709
661 710 return
711
712
713 class SendToFTP(Operation, Process):
714
715 '''
716 Operation to send data over FTP.
717 '''
718
719 __attrs__ = ['server', 'username', 'password', 'patterns', 'timeout']
720
721 def __init__(self, **kwargs):
722 '''
723 patterns = [(local1, remote1, ext, delay, exp_code, sub_exp_code), ...]
724 '''
725 Operation.__init__(self, **kwargs)
726 Process.__init__(self)
727 self.server = kwargs.get('server')
728 self.username = kwargs.get('username')
729 self.password = kwargs.get('password')
730 self.patterns = kwargs.get('patterns')
731 self.timeout = kwargs.get('timeout', 30)
732 self.times = [time.time() for p in self.patterns]
733 self.latest = ['' for p in self.patterns]
734 self.mp = False
735 self.ftp = None
736
737 def setup(self):
738
739 log.log('Connecting to ftp://{}'.format(self.server), self.name)
740 try:
741 self.ftp = ftplib.FTP(self.server, timeout=self.timeout)
742 except ftplib.all_errors:
743 log.error('Server connection fail: {}'.format(self.server), self.name)
744 if self.ftp is not None:
745 self.ftp.close()
746 self.ftp = None
747 self.isConfig = False
748 return
749
750 try:
751 self.ftp.login(self.username, self.password)
752 except ftplib.all_errors:
753 log.error('The given username y/o password are incorrect', self.name)
754 if self.ftp is not None:
755 self.ftp.close()
756 self.ftp = None
757 self.isConfig = False
758 return
759
760 log.success('Connection success', self.name)
761 self.isConfig = True
762 return
763
764 def check(self):
765
766 try:
767 self.ftp.voidcmd("NOOP")
768 except:
769 log.warning('Connection lost... trying to reconnect', self.name)
770 if self.ftp is not None:
771 self.ftp.close()
772 self.ftp = None
773 self.setup()
774
775 def find_files(self, path, ext):
776
777 files = glob.glob1(path, '*{}'.format(ext))
778 files.sort()
779 if files:
780 return files[-1]
781 return None
782
783 def getftpname(self, filename, exp_code, sub_exp_code):
784
785 thisDatetime = datetime.datetime.strptime(filename.split('_')[1], '%Y%m%d')
786 YEAR_STR = '%4.4d'%thisDatetime.timetuple().tm_year
787 DOY_STR = '%3.3d'%thisDatetime.timetuple().tm_yday
788 exp_code = '%3.3d'%exp_code
789 sub_exp_code = '%2.2d'%sub_exp_code
790 plot_code = '%2.2d'% get_plot_code(filename)
791 name = YEAR_STR + DOY_STR + '00' + exp_code + sub_exp_code + plot_code + '00.png'
792 return name
793
794 def upload(self, src, dst):
795
796 log.log('Uploading {} '.format(src), self.name, nl=False)
797
798 fp = open(src, 'rb')
799 command = 'STOR {}'.format(dst)
800
801 try:
802 self.ftp.storbinary(command, fp, blocksize=1024)
803 except Exception, e:
804 log.error('{}'.format(e), self.name)
805 if self.ftp is not None:
806 self.ftp.close()
807 self.ftp = None
808 return 0
809
810 try:
811 self.ftp.sendcmd('SITE CHMOD 755 {}'.format(dst))
812 except Exception, e:
813 log.error('{}'.format(e), self.name)
814 if self.ftp is not None:
815 self.ftp.close()
816 self.ftp = None
817 return 0
818
819 fp.close()
820 log.success('OK', tag='')
821 return 1
822
823 def send_files(self):
824
825 for x, pattern in enumerate(self.patterns):
826 local, remote, ext, delay, exp_code, sub_exp_code = pattern
827 if time.time()-self.times[x] >= delay:
828 srcname = self.find_files(local, ext)
829 src = os.path.join(local, srcname)
830 if os.path.getmtime(src) < time.time() - 30*60:
831 continue
832
833 if srcname is None or srcname == self.latest[x]:
834 continue
835
836 if 'png' in ext:
837 dstname = self.getftpname(srcname, exp_code, sub_exp_code)
838 else:
839 dstname = srcname
840
841 dst = os.path.join(remote, dstname)
842
843 if self.upload(src, dst):
844 self.times[x] = time.time()
845 self.latest[x] = srcname
846 else:
847 self.isConfig = False
848 break
849
850 def run(self):
851
852 while True:
853 if not self.isConfig:
854 self.setup()
855 if self.ftp is not None:
856 self.check()
857 self.send_files()
858 time.sleep(10)
859
860 def close():
861
862 if self.ftp is not None:
863 self.ftp.close()
864 self.terminate()
Show file before | Show file after | Hide comments
@@ -18,23 +18,35 SCHAINPY - LOG
18 18 import click
19 19
20 20
21 def warning(message, tag='Warning'):
22 click.echo(click.style('[{}] {}'.format(tag, message), fg='yellow'))
21 def warning(message, tag='Warning', nl=True):
22 if tag:
23 click.echo(click.style('[{}] {}'.format(tag, message), fg='yellow'), nl=nl)
24 else:
25 click.echo(click.style('{}'.format(message), fg='yellow'), nl=nl)
23 26 pass
24 27
25 28
26 def error(message, tag='Error'):
27 click.echo(click.style('[{}] {}'.format(tag, message), fg='red'))
29 def error(message, tag='Error', nl=True):
30 if tag:
31 click.echo(click.style('[{}] {}'.format(tag, message), fg='red'), nl=nl)
32 else:
33 click.echo(click.style('{}'.format(message), fg='red'), nl=nl)
28 34 pass
29 35
30 36
31 def success(message, tag='Info'):
32 click.echo(click.style('[{}] {}'.format(tag, message), fg='green'))
37 def success(message, tag='Success', nl=True):
38 if tag:
39 click.echo(click.style('[{}] {}'.format(tag, message), fg='green'), nl=nl)
40 else:
41 click.echo(click.style('{}'.format(message), fg='green'), nl=nl)
33 42 pass
34 43
35 44
36 def log(message, tag='Info'):
37 click.echo('[{}] {}'.format(tag, message))
45 def log(message, tag='Info', nl=True):
46 if tag:
47 click.echo('[{}] {}'.format(tag, message), nl=nl)
48 else:
49 click.echo('{}'.format(message), nl=nl)
38 50 pass
39 51
40 52
General Comments 0
You need to be logged in to leave comments. Login now