##// END OF EJS Templates
schain-jc
RSS
NCDFReader -> PXReader, add PXParametersProc to select ppi or rhi modes
jespinoza -
r1140:cd2e2ed64989
parent child
Show More
Side by Side Unified
Context file:
r1140:cd2e2ed64989 Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,24
1 """
2 Utilities for IO modules
3 """
4
5 import os
6 from datetime import datetime
7
8 def folder_in_range(folder, start_date, end_date, pattern):
9 """
10 Check whether folder is bettwen start_date and end_date
11
12 Args:
13 folder (str): Folder to check
14 start_date (date): Initial date
15 end_date (date): Final date
16 pattern (str): Datetime format of the folder
17 Returns:
18 bool: True for success, False otherwise
19 """
20 try:
21 dt = datetime.strptime(folder, pattern)
22 except:
23 raise ValueError('Folder {} does not match {} format'.format(folder, pattern))
24 return start_date <= dt.date() <= end_date
Show file before | Show file after | Hide comments
@@ -0,0 +1,64
1 '''
2 Created on Oct 24, 2016
3
4 @author: roj- LouVD
5 '''
6
7 import numpy
8 import datetime
9 import time
10 from time import gmtime
11
12 from numpy import transpose
13
14 from jroproc_base import ProcessingUnit, Operation
15 from schainpy.model.data.jrodata import Parameters
16
17
18 class PXParametersProc(ProcessingUnit):
19 '''
20 Processing unit for PX parameters data
21 '''
22
23 def __init__(self, **kwargs):
24 """
25 Inputs: None
26 """
27 ProcessingUnit.__init__(self, **kwargs)
28 self.dataOut = Parameters()
29 self.isConfig = False
30
31 def setup(self, mode):
32 """
33 """
34 self.dataOut.mode = mode
35
36 def run(self, mode):
37 """
38 Args:
39 mode (str): select independent variable 'E' for elevation or 'A' for azimuth
40 """
41
42 if not self.isConfig:
43 self.setup(mode)
44 self.isConfig = True
45
46 if self.dataIn.type == 'Parameters':
47 self.dataOut.copy(self.dataIn)
48
49 self.dataOut.data_param = numpy.array([self.dataOut.data[var] for var in self.dataOut.parameters])
50 self.dataOut.data_param[self.dataOut.data_param == self.dataOut.missing] = numpy.nan
51
52 if mode.upper()=='E':
53 self.dataOut.heightList = self.dataOut.data['Azimuth']
54 else:
55 self.dataOut.heightList = self.dataOut.data['Elevation']
56
57 attrs = ['units', 'elevation', 'azimuth', 'max_range']
58 meta = {}
59
60 for attr in attrs:
61 meta[attr] = getattr(self.dataOut, attr)
62
63 meta['mode'] = mode
64 self.dataOut.meta = meta No newline at end of file
Show file before | Show file after | Hide comments
@@ -15,6 +15,8 import tarfile
15 import numpy
15 import numpy
16 from netCDF4 import Dataset
16 from netCDF4 import Dataset
17
17
18 from utils import folder_in_range
19
18 from schainpy.model.io.jroIO_base import JRODataReader
20 from schainpy.model.io.jroIO_base import JRODataReader
19 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
21 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
20 from schainpy.model.data.jrodata import Parameters
22 from schainpy.model.data.jrodata import Parameters
@@ -41,7 +43,7 def load_json(obj):
41 return iterable
43 return iterable
42
44
43
45
44 class NCDFReader(JRODataReader, ProcessingUnit):
46 class PXReader(JRODataReader, ProcessingUnit):
45
47
46 def __init__(self, **kwargs):
48 def __init__(self, **kwargs):
47
49
@@ -98,7 +100,7 class NCDFReader(JRODataReader, ProcessingUnit):
98 path - Path to find files
100 path - Path to find files
99 '''
101 '''
100
102
101 log.log('Searching files {} in {} '.format(self.ext, path), 'NCDFReader')
103 log.log('Searching files {} in {} '.format(self.ext, path), 'PXReader')
102 if walk:
104 if walk:
103 paths = [os.path.join(path, p) for p in os.listdir(path) if os.path.isdir(os.path.join(path, p))]
105 paths = [os.path.join(path, p) for p in os.listdir(path) if os.path.isdir(os.path.join(path, p))]
104 paths.sort()
106 paths.sort()
@@ -108,7 +110,9 class NCDFReader(JRODataReader, ProcessingUnit):
108 fileList0 = []
110 fileList0 = []
109
111
110 for subpath in paths:
112 for subpath in paths:
111 fileList0 += [os.path.join(subpath, s) for s in glob.glob1(subpath, '*') if os.path.splitext(s)[-1] in self.ext and 'E{}'.format(self.ele) in s]
113 if not folder_in_range(subpath.split('/')[-1], startDate, endDate, '%Y%m%d'):
114 continue
115 fileList0 += [os.path.join(subpath, s) for s in glob.glob1(subpath, '*') if os.path.splitext(s)[-1] in self.ext and '{}'.format(self.ele) in s]
112
116
113 fileList0.sort()
117 fileList0.sort()
114 if self.online:
118 if self.online:
@@ -169,7 +173,7 class NCDFReader(JRODataReader, ProcessingUnit):
169 else:
173 else:
170 paths = self.path
174 paths = self.path
171
175
172 new_files = [os.path.join(path, s) for s in glob.glob1(path, '*') if os.path.splitext(s)[-1] in self.ext and 'E{}'.format(self.ele) in s]
176 new_files = [os.path.join(path, s) for s in glob.glob1(path, '*') if os.path.splitext(s)[-1] in self.ext and '{}'.format(self.ele) in s]
173
177
174 new_files.sort()
178 new_files.sort()
175
179
@@ -206,7 +210,7 class NCDFReader(JRODataReader, ProcessingUnit):
206 if self.files:
210 if self.files:
207 break
211 break
208 else:
212 else:
209 log.warning('Waiting {} seconds for the next file, try {} ...'.format(self.delay, n + 1), 'NCDFReader')
213 log.warning('Waiting {} seconds for the next file, try {} ...'.format(self.delay, n + 1), 'PXReader')
210 time.sleep(self.delay)
214 time.sleep(self.delay)
211
215
212 if not self.files:
216 if not self.files:
@@ -222,16 +226,14 class NCDFReader(JRODataReader, ProcessingUnit):
222 '''
226 '''
223 '''
227 '''
224
228
225 self.header = {}
229 header = {}
226
230
227 for attr in self.fp.ncattrs():
231 for attr in self.fp.ncattrs():
228 self.header[str(attr)] = getattr(self.fp, attr)
232 header[str(attr)] = getattr(self.fp, attr)
229
233
230 self.data[self.header['TypeName']] = numpy.array(self.fp.variables[self.header['TypeName']])
234 self.header.append(header)
231
235
232 if 'Azimuth' not in self.data:
236 self.data[header['TypeName']] = numpy.array(self.fp.variables[header['TypeName']])
233 self.data['Azimuth'] = numpy.array(self.fp.variables['Azimuth'])
234
235
237
236 def setNextFile(self):
238 def setNextFile(self):
237 '''
239 '''
@@ -246,26 +248,24 class NCDFReader(JRODataReader, ProcessingUnit):
246 self.dt = self.dates[cursor]
248 self.dt = self.dates[cursor]
247 self.online_mode = True
249 self.online_mode = True
248 if not self.search_files_online():
250 if not self.search_files_online():
249 log.success('No more files', 'NCDFReader')
251 log.success('No more files', 'PXReader')
250 return 0
252 return 0
251 else:
253 else:
252 log.success('No more files', 'NCDFReader')
254 log.success('No more files', 'PXReader')
253 self.flagNoMoreFiles = 1
255 self.flagNoMoreFiles = 1
254 return 0
256 return 0
255 else:
257 else:
256 if not self.search_files_online():
258 if not self.search_files_online():
257 return 0
259 return 0
258 cursor = self.cursor
260 cursor = self.cursor
259
260 log.log(
261 'Opening: {}\'s files'.format(self.dates[cursor]),
262 'NCDFReader'
263 )
264
261
265 self.data = {}
262 self.data = {}
263 self.header = []
266
264
267 for fullname in self.files[self.dates[cursor]]:
265 for fullname in self.files[self.dates[cursor]]:
268
266
267 log.log('Opening: {}'.format(fullname), 'PXReader')
268
269 if os.path.splitext(fullname)[-1] == '.tgz':
269 if os.path.splitext(fullname)[-1] == '.tgz':
270 tar = tarfile.open(fullname, 'r:gz')
270 tar = tarfile.open(fullname, 'r:gz')
271 tar.extractall('/tmp')
271 tar.extractall('/tmp')
@@ -293,7 +293,7 class NCDFReader(JRODataReader, ProcessingUnit):
293 if not self.setNextFile():
293 if not self.setNextFile():
294 return 0
294 return 0
295
295
296 self.datatime = datetime.datetime.utcfromtimestamp(self.header['Time'])
296 self.datatime = datetime.datetime.utcfromtimestamp(self.header[0]['Time'])
297
297
298 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
298 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
299 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
299 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
@@ -302,7 +302,7 class NCDFReader(JRODataReader, ProcessingUnit):
302 self.counter_records,
302 self.counter_records,
303 self.nrecords,
303 self.nrecords,
304 self.datatime.ctime()),
304 self.datatime.ctime()),
305 'NCDFReader')
305 'PXReader')
306 continue
306 continue
307 break
307 break
308
308
@@ -311,7 +311,7 class NCDFReader(JRODataReader, ProcessingUnit):
311 self.counter_records,
311 self.counter_records,
312 self.nrecords,
312 self.nrecords,
313 self.datatime.ctime()),
313 self.datatime.ctime()),
314 'NCDFReader')
314 'PXReader')
315
315
316 return 1
316 return 1
317
317
@@ -319,29 +319,36 class NCDFReader(JRODataReader, ProcessingUnit):
319 def set_output(self):
319 def set_output(self):
320 '''
320 '''
321 Storing data from buffer to dataOut object
321 Storing data from buffer to dataOut object
322 '''
322 '''
323
323
324 self.dataOut.heightList = self.data.pop('Azimuth')
324 self.data['Elevation'] = numpy.array(self.fp.variables['Elevation'])
325
325 self.data['Azimuth'] = numpy.array(self.fp.variables['Azimuth'])
326 log.log('Parameters found: {}'.format(','.join(self.data.keys())),
326 self.dataOut.range = numpy.array(self.fp.variables['GateWidth'])
327 'PXReader')
327 self.dataOut.data = self.data
328
328 self.dataOut.units = [h['Unit-value'] for h in self.header]
329 self.dataOut.data_param = numpy.array(self.data.values())
329 self.dataOut.parameters = [h['TypeName'] for h in self.header]
330 self.dataOut.data_param[self.dataOut.data_param == -99900.] = numpy.nan
330 self.dataOut.missing = self.header[0]['MissingData']
331 self.dataOut.parameters = self.data.keys()
331 self.dataOut.max_range = self.header[0]['MaximumRange-value']
332 self.dataOut.utctime = self.header['Time']
332 self.dataOut.elevation = self.header[0]['Elevation']
333 self.dataOut.azimuth = self.header[0]['Azimuth']
334 self.dataOut.latitude = self.header[0]['Latitude']
335 self.dataOut.longitude = self.header[0]['Longitude']
336 self.dataOut.utctime = self.header[0]['Time']
333 self.dataOut.utctimeInit = self.dataOut.utctime
337 self.dataOut.utctimeInit = self.dataOut.utctime
334 self.dataOut.useLocalTime = False
338 self.dataOut.useLocalTime = True
335 self.dataOut.flagNoData = False
339 self.dataOut.flagNoData = False
336 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
340 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
337
341
342 log.log('Parameters found: {}'.format(','.join(self.dataOut.parameters)),
343 'PXReader')
344
338 def getData(self):
345 def getData(self):
339 '''
346 '''
340 Storing data from databuffer to dataOut object
347 Storing data from databuffer to dataOut object
341 '''
348 '''
342 if self.flagNoMoreFiles:
349 if self.flagNoMoreFiles:
343 self.dataOut.flagNoData = True
350 self.dataOut.flagNoData = True
344 log.error('No file left to process', 'NCDFReader')
351 log.error('No file left to process', 'PXReader')
345 return 0
352 return 0
346
353
347 if not self.readNextFile():
354 if not self.readNextFile():
Show file before | Show file after | Hide comments
@@ -13,3 +13,4 from jroproc_parameters import *
13 from jroproc_spectra_lags import *
13 from jroproc_spectra_lags import *
14 from jroproc_spectra_acf import *
14 from jroproc_spectra_acf import *
15 from bltrproc_parameters import *
15 from bltrproc_parameters import *
16 from pxproc_parameters import *
General Comments 0
You need to be logged in to leave comments. Login now