##// END OF EJS Templates
schain-jc
RSS
Add NCDFReader for PX1000 radar, new PlotPolarMap, bugs in jroplot_data
jespinoza -
r1137:124e67b869ed
parent child
Show More
Side by Side Unified
Context file:
r1137:124e67b869ed Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,351
1 '''
2 Created on Dec 27, 2017
3
4 @author: Juan C. Espinoza
5 '''
6
7 import os
8 import sys
9 import time
10 import json
11 import glob
12 import datetime
13 import tarfile
14
15 import numpy
16 from netCDF4 import Dataset
17
18 from schainpy.model.io.jroIO_base import JRODataReader
19 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
20 from schainpy.model.data.jrodata import Parameters
21 from schainpy.utils import log
22
23 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
24
25 def load_json(obj):
26 '''
27 Parse json as string instead of unicode
28 '''
29
30 if isinstance(obj, str):
31 iterable = json.loads(obj)
32 else:
33 iterable = obj
34
35 if isinstance(iterable, dict):
36 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
37 for k, v in iterable.items()}
38 elif isinstance(iterable, (list, tuple)):
39 return [str(v) if isinstance(v, unicode) else v for v in iterable]
40
41 return iterable
42
43
44 class NCDFReader(JRODataReader, ProcessingUnit):
45
46 def __init__(self, **kwargs):
47
48 ProcessingUnit.__init__(self, **kwargs)
49
50 self.dataOut = Parameters()
51 self.counter_records = 0
52 self.nrecords = None
53 self.flagNoMoreFiles = 0
54 self.isConfig = False
55 self.filename = None
56 self.intervals = set()
57 self.ext = ('.nc', '.tgz')
58 self.online_mode = False
59
60 def setup(self,
61 path=None,
62 startDate=None,
63 endDate=None,
64 format=None,
65 startTime=datetime.time(0, 0, 0),
66 endTime=datetime.time(23, 59, 59),
67 walk=False,
68 **kwargs):
69
70 self.path = path
71 self.startDate = startDate
72 self.endDate = endDate
73 self.startTime = startTime
74 self.endTime = endTime
75 self.datatime = datetime.datetime(1900,1,1)
76 self.walk = walk
77 self.nTries = kwargs.get('nTries', 3)
78 self.online = kwargs.get('online', False)
79 self.delay = kwargs.get('delay', 30)
80
81 if self.path is None:
82 raise ValueError, 'The path is not valid'
83
84 self.search_files(path, startDate, endDate, startTime, endTime, walk)
85 self.cursor = 0
86 self.counter_records = 0
87
88 if not self.files:
89 raise Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path)
90
91 def search_files(self, path, startDate, endDate, startTime, endTime, walk):
92 '''
93 Searching for NCDF files in path
94 Creating a list of files to procces included in [startDate,endDate]
95
96 Input:
97 path - Path to find files
98 '''
99
100 log.log('Searching files {} in {} '.format(self.ext, path), 'NCDFReader')
101 if walk:
102 paths = [os.path.join(path, p) for p in os.listdir(path) if os.path.isdir(os.path.join(path, p))]
103 paths.sort()
104 else:
105 paths = [path]
106
107 fileList0 = []
108
109 for subpath in paths:
110 fileList0 += [os.path.join(subpath, s) for s in glob.glob1(subpath, '*') if os.path.splitext(s)[-1] in self.ext]
111
112 fileList0.sort()
113
114 self.files = {}
115
116 startDate = startDate - datetime.timedelta(1)
117 endDate = endDate + datetime.timedelta(1)
118
119 for fullname in fileList0:
120 thisFile = fullname.split('/')[-1]
121 year = thisFile[3:7]
122 if not year.isdigit():
123 continue
124
125 month = thisFile[7:9]
126 if not month.isdigit():
127 continue
128
129 day = thisFile[9:11]
130 if not day.isdigit():
131 continue
132
133 year, month, day = int(year), int(month), int(day)
134 dateFile = datetime.date(year, month, day)
135 timeFile = datetime.time(int(thisFile[12:14]), int(thisFile[14:16]), int(thisFile[16:18]))
136
137 if (startDate > dateFile) or (endDate < dateFile):
138 continue
139
140 dt = datetime.datetime.combine(dateFile, timeFile)
141 if dt not in self.files:
142 self.files[dt] = []
143 self.files[dt].append(fullname)
144
145 self.dates = self.files.keys()
146 self.dates.sort()
147
148 return
149
150 def search_files_online(self):
151 '''
152 Searching for NCDF files in online mode path
153 Creating a list of files to procces included in [startDate,endDate]
154
155 Input:
156 path - Path to find files
157 '''
158
159 old_files = self.files[self.dt]
160 self.files = {}
161
162 for n in range(self.nTries):
163
164 if self.walk:
165 paths = [os.path.join(self.path, p) for p in os.listdir(self.path) if os.path.isdir(os.path.join(self.path, p))]
166 paths = paths[-2:]
167 else:
168 paths = [self.path]
169
170 new_files = []
171
172 for path in paths:
173 new_files += [os.path.join(path, s) for s in glob.glob1(path, '*') if os.path.splitext(s)[-1] in self.ext and os.path.join(path, s not in old_files)]
174
175 new_files.sort()
176
177 if new_files:
178 break
179 else:
180 log.warning('Waiting {} seconds for the next file, try {} ...'.format(self.delay, n + 1), 'NCDFReader')
181 time.sleep(self.delay)
182
183 if not new_files:
184 log.error('No more files found', 'NCDFReader')
185 return 0
186
187 startDate = self.dt - datetime.timedelta(seconds=1)
188
189 for fullname in new_files:
190 thisFile = fullname.split('/')[-1]
191 year = thisFile[3:7]
192 if not year.isdigit():
193 continue
194
195 month = thisFile[7:9]
196 if not month.isdigit():
197 continue
198
199 day = thisFile[9:11]
200 if not day.isdigit():
201 continue
202
203 year, month, day = int(year), int(month), int(day)
204 dateFile = datetime.date(year, month, day)
205 timeFile = datetime.time(int(thisFile[12:14]), int(thisFile[14:16]), int(thisFile[16:18]))
206
207 if (startDate > dateFile):
208 continue
209
210 dt = datetime.datetime.combine(dateFile, timeFile)
211 if dt not in self.files:
212 self.files[dt] = []
213
214 self.files[dt].append(fullname)
215
216 self.dates = self.files.keys()
217 self.dates.sort()
218 self.cursor = 0
219
220 return 1
221
222 def parseFile(self):
223 '''
224 '''
225
226 self.header = {}
227
228 for attr in self.fp.ncattrs():
229 self.header[str(attr)] = getattr(self.fp, attr)
230
231 self.data[self.header['TypeName']] = numpy.array(self.fp.variables[self.header['TypeName']])
232
233 if 'Azimuth' not in self.data:
234 self.data['Azimuth'] = numpy.array(self.fp.variables['Azimuth'])
235
236
237 def setNextFile(self):
238 '''
239 Open next files for the current datetime
240 '''
241
242 cursor = self.cursor
243
244 if not self.online_mode:
245 self.dt = self.dates[cursor]
246 if cursor == len(self.dates):
247 if self.online:
248 self.online_mode = True
249 else:
250 log.success('No more files', 'NCDFReader')
251 self.flagNoMoreFiles = 1
252 return 0
253 else:
254 if not self.search_files_online():
255 return 0
256
257 log.log(
258 'Opening: {}\'s files'.format(self.dates[cursor]),
259 'NCDFReader'
260 )
261
262 self.data = {}
263
264 for fullname in self.files[self.dates[cursor]]:
265
266 if os.path.splitext(fullname)[-1] == '.tgz':
267 tar = tarfile.open(fullname, 'r:gz')
268 tar.extractall('/tmp')
269 files = [os.path.join('/tmp', member.name) for member in tar.getmembers()]
270 else:
271 files = [fullname]
272
273 for filename in files:
274 if self.filename is not None:
275 self.fp.close()
276
277 self.filename = filename
278 self.filedate = self.dates[cursor]
279 self.fp = Dataset(self.filename, 'r')
280 self.parseFile()
281
282 self.counter_records += 1
283 self.cursor += 1
284 return 1
285
286 def readNextFile(self):
287
288 while True:
289 self.flagDiscontinuousBlock = 0
290 if not self.setNextFile():
291 return 0
292
293 self.datatime = datetime.datetime.utcfromtimestamp(self.header['Time'])
294
295 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
296 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
297 log.warning(
298 'Reading Record No. {}/{} -> {} [Skipping]'.format(
299 self.counter_records,
300 self.nrecords,
301 self.datatime.ctime()),
302 'NCDFReader')
303 continue
304 break
305
306 log.log(
307 'Reading Record No. {}/{} -> {}'.format(
308 self.counter_records,
309 self.nrecords,
310 self.datatime.ctime()),
311 'NCDFReader')
312
313 return 1
314
315
316 def set_output(self):
317 '''
318 Storing data from buffer to dataOut object
319 '''
320
321 self.dataOut.heightList = self.data.pop('Azimuth')
322
323 log.log('Parameters found: {}'.format(','.join(self.data.keys())),
324 'PXReader')
325
326 self.dataOut.data_param = numpy.array(self.data.values())
327 self.dataOut.data_param[self.dataOut.data_param == -99900.] = numpy.nan
328 self.dataOut.parameters = self.data.keys()
329 self.dataOut.utctime = self.header['Time']
330 self.dataOut.utctimeInit = self.dataOut.utctime
331 self.dataOut.useLocalTime = False
332 self.dataOut.flagNoData = False
333 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
334
335 def getData(self):
336 '''
337 Storing data from databuffer to dataOut object
338 '''
339 if self.flagNoMoreFiles:
340 self.dataOut.flagNoData = True
341 log.error('No file left to process', 'NCDFReader')
342 return 0
343
344 if not self.readNextFile():
345 self.dataOut.flagNoData = True
346 return 0
347
348 self.set_output()
349
350 return 1
351
Show file before | Show file after | Hide comments
@@ -22,7 +22,7 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
22 22 'jro', numpy.vstack((blu_values, jet_values)))
23 23 matplotlib.pyplot.register_cmap(cmap=ncmap)
24 24
25 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis', 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
25 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis', 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm', 'spectral')]
26 26
27 27
28 28 def figpause(interval):
@@ -51,7 +51,7 class PlotData(Operation, Process):
51 51 __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
52 52 'zlimits', 'xlabel', 'ylabel', 'xaxis','cb_label', 'title',
53 53 'colorbar', 'bgcolor', 'width', 'height', 'localtime', 'oneFigure',
54 'showprofile', 'decimation']
54 'showprofile', 'decimation', 'ftp']
55 55
56 56 def __init__(self, **kwargs):
57 57
@@ -68,6 +68,7 class PlotData(Operation, Process):
68 68 self.localtime = kwargs.pop('localtime', True)
69 69 self.show = kwargs.get('show', True)
70 70 self.save = kwargs.get('save', False)
71 self.ftp = kwargs.get('ftp', False)
71 72 self.colormap = kwargs.get('colormap', self.colormap)
72 73 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
73 74 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
@@ -77,6 +78,7 class PlotData(Operation, Process):
77 78 self.title = kwargs.get('wintitle', self.CODE.upper())
78 79 self.cb_label = kwargs.get('cb_label', None)
79 80 self.cb_labels = kwargs.get('cb_labels', None)
81 self.labels = kwargs.get('labels', None)
80 82 self.xaxis = kwargs.get('xaxis', 'frequency')
81 83 self.zmin = kwargs.get('zmin', None)
82 84 self.zmax = kwargs.get('zmax', None)
@@ -84,8 +86,10 class PlotData(Operation, Process):
84 86 self.xmin = kwargs.get('xmin', None)
85 87 self.xmax = kwargs.get('xmax', None)
86 88 self.xrange = kwargs.get('xrange', 24)
89 self.xscale = kwargs.get('xscale', None)
87 90 self.ymin = kwargs.get('ymin', None)
88 91 self.ymax = kwargs.get('ymax', None)
92 self.yscale = kwargs.get('yscale', None)
89 93 self.xlabel = kwargs.get('xlabel', None)
90 94 self.decimation = kwargs.get('decimation', None)
91 95 self.showSNR = kwargs.get('showSNR', False)
@@ -94,6 +98,7 class PlotData(Operation, Process):
94 98 self.height = kwargs.get('height', None)
95 99 self.colorbar = kwargs.get('colorbar', True)
96 100 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
101 self.channels = kwargs.get('channels', None)
97 102 self.titles = kwargs.get('titles', [])
98 103 self.polar = False
99 104
@@ -368,14 +373,19 class PlotData(Operation, Process):
368 373 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
369 374 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
370 375
371 Y = numpy.array([5, 10, 20, 50, 100, 200, 500, 1000, 2000])
372 i = 1 if numpy.where(ymax-ymin < Y)[0][0] < 0 else numpy.where(ymax-ymin < Y)[0][0]
373 ystep = Y[i] / 5
376 Y = numpy.array([5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000])
377 i = 1 if numpy.where(abs(ymax-ymin) <= Y)[0][0] < 0 else numpy.where(abs(ymax-ymin) <= Y)[0][0]
378 ystep = Y[i] / 5.
374 379
375 380 for n, ax in enumerate(self.axes):
376 381 if ax.firsttime:
377 382 ax.set_facecolor(self.bgcolor)
378 383 ax.yaxis.set_major_locator(MultipleLocator(ystep))
384 ax.xaxis.set_major_locator(MultipleLocator(ystep))
385 if self.xscale:
386 ax.xaxis.set_major_formatter(FuncFormatter(lambda x, pos: '{0:g}'.format(x*self.xscale)))
387 if self.xscale:
388 ax.yaxis.set_major_formatter(FuncFormatter(lambda x, pos: '{0:g}'.format(x*self.yscale)))
379 389 if self.xaxis is 'time':
380 390 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
381 391 ax.xaxis.set_major_locator(LinearLocator(9))
@@ -419,7 +429,7 class PlotData(Operation, Process):
419 429 def __plot(self):
420 430 '''
421 431 '''
422 log.success('Plotting', self.name)
432 log.log('Plotting', self.name)
423 433
424 434 try:
425 435 self.plot()
@@ -439,14 +449,20 class PlotData(Operation, Process):
439 449 self.getDateTime(self.max_time).strftime('%Y/%m/%d')))
440 450 fig.canvas.draw()
441 451
442 if self.save and self.data.ended:
443 channels = range(self.nrows)
452 if self.save and (self.data.ended or not self.data.buffering):
453
454 if self.labels:
455 labels = self.labels
456 else:
457 labels = range(self.nrows)
458
444 459 if self.oneFigure:
445 460 label = ''
446 461 else:
447 label = '_{}'.format(channels[n])
462 label = '-{}'.format(labels[n])
448 463 figname = os.path.join(
449 464 self.save,
465 self.CODE,
450 466 '{}{}_{}.png'.format(
451 467 self.CODE,
452 468 label,
@@ -455,6 +471,8 class PlotData(Operation, Process):
455 471 )
456 472 )
457 473 log.log('Saving figure: {}'.format(figname), self.name)
474 if not os.path.isdir(os.path.dirname(figname)):
475 os.makedirs(os.path.dirname(figname))
458 476 fig.savefig(figname)
459 477
460 478 def plot(self):
@@ -464,7 +482,7 class PlotData(Operation, Process):
464 482
465 483 def run(self):
466 484
467 log.success('Starting', self.name)
485 log.log('Starting', self.name)
468 486
469 487 context = zmq.Context()
470 488 receiver = context.socket(zmq.SUB)
@@ -499,7 +517,7 class PlotData(Operation, Process):
499 517 self.__plot()
500 518
501 519 except zmq.Again as e:
502 log.log('Waiting for data...')
520 log.log('.', tag='', nl=False)
503 521 if self.data:
504 522 figpause(self.data.throttle)
505 523 else:
@@ -963,3 +981,62 class PlotOutputData(PlotParamData):
963 981
964 982 CODE = 'output'
965 983 colormap = 'seismic'
984
985
986 class PlotPolarMapData(PlotData):
987 '''
988 Plot for meteors detection data
989 '''
990
991 CODE = 'param'
992 colormap = 'seismic'
993
994 def setup(self):
995 self.ncols = 1
996 self.nrows = 1
997 self.width = 9
998 self.height = 8
999 if self.channels is not None:
1000 self.nplots = len(self.channels)
1001 self.nrows = len(self.channels)
1002 else:
1003 self.nplots = self.data.shape(self.CODE)[0]
1004 self.nrows = self.nplots
1005 self.channels = range(self.nplots)
1006 self.xlabel = 'Zonal Distance (km)'
1007 self.ylabel = 'Meridional Distance (km)'
1008 self.bgcolor = 'white'
1009
1010 def plot(self):
1011
1012 for n, ax in enumerate(self.axes):
1013 data = self.data['param'][self.channels[n]]
1014
1015 zeniths = numpy.arange(data.shape[1])
1016 azimuths = -numpy.radians(self.data.heights)+numpy.pi/2
1017 self.y = zeniths
1018
1019 r, theta = numpy.meshgrid(zeniths, azimuths)
1020 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
1021
1022 if ax.firsttime:
1023 if self.zlimits is not None:
1024 self.zmin, self.zmax = self.zlimits[n]
1025 ax.plt = ax.pcolormesh(x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
1026 vmin=self.zmin,
1027 vmax=self.zmax,
1028 cmap=self.cmaps[n])
1029 else:
1030 if self.zlimits is not None:
1031 self.zmin, self.zmax = self.zlimits[n]
1032 ax.collections.remove(ax.collections[0])
1033 ax.plt = ax.pcolormesh(x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
1034 vmin=self.zmin,
1035 vmax=self.zmax,
1036 cmap=self.cmaps[n])
1037
1038
1039 title = ''
1040
1041 self.titles = [self.data.parameters[x] for x in self.channels]
1042 self.saveTime = self.max_time
Show file before | Show file after | Hide comments
@@ -19,3 +19,5 from bltrIO_param import *
19 19 from jroIO_bltr import *
20 20 from jroIO_mira35c import *
21 21 from julIO_param import *
22
23 from pxIO_param import * No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now