##// END OF EJS Templates
schain-jc
RSS
NCDFReader -> PXReader, add PXParametersProc to select ppi or rhi modes
jespinoza -
r1140:cd2e2ed64989
parent child
Show More
Side by Side Unified
Context file:
r1140:cd2e2ed64989 Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,24
1 """
2 Utilities for IO modules
3 """
4
5 import os
6 from datetime import datetime
7
8 def folder_in_range(folder, start_date, end_date, pattern):
9 """
10 Check whether folder is bettwen start_date and end_date
11
12 Args:
13 folder (str): Folder to check
14 start_date (date): Initial date
15 end_date (date): Final date
16 pattern (str): Datetime format of the folder
17 Returns:
18 bool: True for success, False otherwise
19 """
20 try:
21 dt = datetime.strptime(folder, pattern)
22 except:
23 raise ValueError('Folder {} does not match {} format'.format(folder, pattern))
24 return start_date <= dt.date() <= end_date
Show file before | Show file after | Hide comments
@@ -0,0 +1,64
1 '''
2 Created on Oct 24, 2016
3
4 @author: roj- LouVD
5 '''
6
7 import numpy
8 import datetime
9 import time
10 from time import gmtime
11
12 from numpy import transpose
13
14 from jroproc_base import ProcessingUnit, Operation
15 from schainpy.model.data.jrodata import Parameters
16
17
18 class PXParametersProc(ProcessingUnit):
19 '''
20 Processing unit for PX parameters data
21 '''
22
23 def __init__(self, **kwargs):
24 """
25 Inputs: None
26 """
27 ProcessingUnit.__init__(self, **kwargs)
28 self.dataOut = Parameters()
29 self.isConfig = False
30
31 def setup(self, mode):
32 """
33 """
34 self.dataOut.mode = mode
35
36 def run(self, mode):
37 """
38 Args:
39 mode (str): select independent variable 'E' for elevation or 'A' for azimuth
40 """
41
42 if not self.isConfig:
43 self.setup(mode)
44 self.isConfig = True
45
46 if self.dataIn.type == 'Parameters':
47 self.dataOut.copy(self.dataIn)
48
49 self.dataOut.data_param = numpy.array([self.dataOut.data[var] for var in self.dataOut.parameters])
50 self.dataOut.data_param[self.dataOut.data_param == self.dataOut.missing] = numpy.nan
51
52 if mode.upper()=='E':
53 self.dataOut.heightList = self.dataOut.data['Azimuth']
54 else:
55 self.dataOut.heightList = self.dataOut.data['Elevation']
56
57 attrs = ['units', 'elevation', 'azimuth', 'max_range']
58 meta = {}
59
60 for attr in attrs:
61 meta[attr] = getattr(self.dataOut, attr)
62
63 meta['mode'] = mode
64 self.dataOut.meta = meta No newline at end of file
Show file before | Show file after | Hide comments
@@ -15,6 +15,8 import tarfile
15 15 import numpy
16 16 from netCDF4 import Dataset
17 17
18 from utils import folder_in_range
19
18 20 from schainpy.model.io.jroIO_base import JRODataReader
19 21 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
20 22 from schainpy.model.data.jrodata import Parameters
@@ -41,7 +43,7 def load_json(obj):
41 43 return iterable
42 44
43 45
44 class NCDFReader(JRODataReader, ProcessingUnit):
46 class PXReader(JRODataReader, ProcessingUnit):
45 47
46 48 def __init__(self, **kwargs):
47 49
@@ -98,7 +100,7 class NCDFReader(JRODataReader, ProcessingUnit):
98 100 path - Path to find files
99 101 '''
100 102
101 log.log('Searching files {} in {} '.format(self.ext, path), 'NCDFReader')
103 log.log('Searching files {} in {} '.format(self.ext, path), 'PXReader')
102 104 if walk:
103 105 paths = [os.path.join(path, p) for p in os.listdir(path) if os.path.isdir(os.path.join(path, p))]
104 106 paths.sort()
@@ -108,7 +110,9 class NCDFReader(JRODataReader, ProcessingUnit):
108 110 fileList0 = []
109 111
110 112 for subpath in paths:
111 fileList0 += [os.path.join(subpath, s) for s in glob.glob1(subpath, '*') if os.path.splitext(s)[-1] in self.ext and 'E{}'.format(self.ele) in s]
113 if not folder_in_range(subpath.split('/')[-1], startDate, endDate, '%Y%m%d'):
114 continue
115 fileList0 += [os.path.join(subpath, s) for s in glob.glob1(subpath, '*') if os.path.splitext(s)[-1] in self.ext and '{}'.format(self.ele) in s]
112 116
113 117 fileList0.sort()
114 118 if self.online:
@@ -169,7 +173,7 class NCDFReader(JRODataReader, ProcessingUnit):
169 173 else:
170 174 paths = self.path
171 175
172 new_files = [os.path.join(path, s) for s in glob.glob1(path, '*') if os.path.splitext(s)[-1] in self.ext and 'E{}'.format(self.ele) in s]
176 new_files = [os.path.join(path, s) for s in glob.glob1(path, '*') if os.path.splitext(s)[-1] in self.ext and '{}'.format(self.ele) in s]
173 177
174 178 new_files.sort()
175 179
@@ -206,7 +210,7 class NCDFReader(JRODataReader, ProcessingUnit):
206 210 if self.files:
207 211 break
208 212 else:
209 log.warning('Waiting {} seconds for the next file, try {} ...'.format(self.delay, n + 1), 'NCDFReader')
213 log.warning('Waiting {} seconds for the next file, try {} ...'.format(self.delay, n + 1), 'PXReader')
210 214 time.sleep(self.delay)
211 215
212 216 if not self.files:
@@ -222,16 +226,14 class NCDFReader(JRODataReader, ProcessingUnit):
222 226 '''
223 227 '''
224 228
225 self.header = {}
229 header = {}
226 230
227 231 for attr in self.fp.ncattrs():
228 self.header[str(attr)] = getattr(self.fp, attr)
232 header[str(attr)] = getattr(self.fp, attr)
229 233
230 self.data[self.header['TypeName']] = numpy.array(self.fp.variables[self.header['TypeName']])
231
232 if 'Azimuth' not in self.data:
233 self.data['Azimuth'] = numpy.array(self.fp.variables['Azimuth'])
234
234 self.header.append(header)
235
236 self.data[header['TypeName']] = numpy.array(self.fp.variables[header['TypeName']])
235 237
236 238 def setNextFile(self):
237 239 '''
@@ -246,26 +248,24 class NCDFReader(JRODataReader, ProcessingUnit):
246 248 self.dt = self.dates[cursor]
247 249 self.online_mode = True
248 250 if not self.search_files_online():
249 log.success('No more files', 'NCDFReader')
251 log.success('No more files', 'PXReader')
250 252 return 0
251 253 else:
252 log.success('No more files', 'NCDFReader')
254 log.success('No more files', 'PXReader')
253 255 self.flagNoMoreFiles = 1
254 256 return 0
255 257 else:
256 258 if not self.search_files_online():
257 259 return 0
258 260 cursor = self.cursor
259
260 log.log(
261 'Opening: {}\'s files'.format(self.dates[cursor]),
262 'NCDFReader'
263 )
264 261
265 262 self.data = {}
263 self.header = []
266 264
267 265 for fullname in self.files[self.dates[cursor]]:
268 266
267 log.log('Opening: {}'.format(fullname), 'PXReader')
268
269 269 if os.path.splitext(fullname)[-1] == '.tgz':
270 270 tar = tarfile.open(fullname, 'r:gz')
271 271 tar.extractall('/tmp')
@@ -293,7 +293,7 class NCDFReader(JRODataReader, ProcessingUnit):
293 293 if not self.setNextFile():
294 294 return 0
295 295
296 self.datatime = datetime.datetime.utcfromtimestamp(self.header['Time'])
296 self.datatime = datetime.datetime.utcfromtimestamp(self.header[0]['Time'])
297 297
298 298 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
299 299 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
@@ -302,7 +302,7 class NCDFReader(JRODataReader, ProcessingUnit):
302 302 self.counter_records,
303 303 self.nrecords,
304 304 self.datatime.ctime()),
305 'NCDFReader')
305 'PXReader')
306 306 continue
307 307 break
308 308
@@ -311,7 +311,7 class NCDFReader(JRODataReader, ProcessingUnit):
311 311 self.counter_records,
312 312 self.nrecords,
313 313 self.datatime.ctime()),
314 'NCDFReader')
314 'PXReader')
315 315
316 316 return 1
317 317
@@ -319,29 +319,36 class NCDFReader(JRODataReader, ProcessingUnit):
319 319 def set_output(self):
320 320 '''
321 321 Storing data from buffer to dataOut object
322 '''
323
324 self.dataOut.heightList = self.data.pop('Azimuth')
325
326 log.log('Parameters found: {}'.format(','.join(self.data.keys())),
327 'PXReader')
328
329 self.dataOut.data_param = numpy.array(self.data.values())
330 self.dataOut.data_param[self.dataOut.data_param == -99900.] = numpy.nan
331 self.dataOut.parameters = self.data.keys()
332 self.dataOut.utctime = self.header['Time']
322 '''
323
324 self.data['Elevation'] = numpy.array(self.fp.variables['Elevation'])
325 self.data['Azimuth'] = numpy.array(self.fp.variables['Azimuth'])
326 self.dataOut.range = numpy.array(self.fp.variables['GateWidth'])
327 self.dataOut.data = self.data
328 self.dataOut.units = [h['Unit-value'] for h in self.header]
329 self.dataOut.parameters = [h['TypeName'] for h in self.header]
330 self.dataOut.missing = self.header[0]['MissingData']
331 self.dataOut.max_range = self.header[0]['MaximumRange-value']
332 self.dataOut.elevation = self.header[0]['Elevation']
333 self.dataOut.azimuth = self.header[0]['Azimuth']
334 self.dataOut.latitude = self.header[0]['Latitude']
335 self.dataOut.longitude = self.header[0]['Longitude']
336 self.dataOut.utctime = self.header[0]['Time']
333 337 self.dataOut.utctimeInit = self.dataOut.utctime
334 self.dataOut.useLocalTime = False
338 self.dataOut.useLocalTime = True
335 339 self.dataOut.flagNoData = False
336 340 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
337 341
342 log.log('Parameters found: {}'.format(','.join(self.dataOut.parameters)),
343 'PXReader')
344
338 345 def getData(self):
339 346 '''
340 347 Storing data from databuffer to dataOut object
341 348 '''
342 349 if self.flagNoMoreFiles:
343 350 self.dataOut.flagNoData = True
344 log.error('No file left to process', 'NCDFReader')
351 log.error('No file left to process', 'PXReader')
345 352 return 0
346 353
347 354 if not self.readNextFile():
Show file before | Show file after | Hide comments
@@ -13,3 +13,4 from jroproc_parameters import *
13 13 from jroproc_spectra_lags import *
14 14 from jroproc_spectra_acf import *
15 15 from bltrproc_parameters import *
16 from pxproc_parameters import *
General Comments 0
You need to be logged in to leave comments. Login now