##// END OF EJS Templates
schain-jc
RSS
formatting
José Chávez -
r1083:e4f5cab49468
parent child
Show More
Side by Side Unified
Context file:
r1083:e4f5cab49468 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,362 +1,365
1 '''
1 '''
2 Created on Nov 9, 2016
2 Created on Nov 9, 2016
3
3
4 @author: roj- LouVD
4 @author: roj- LouVD
5 '''
5 '''
6
6
7
7
8 import os
8 import os
9 import sys
9 import sys
10 import time
10 import time
11 import glob
11 import glob
12 import datetime
12 import datetime
13
13
14 import numpy
14 import numpy
15
15
16 from schainpy.model.proc.jroproc_base import ProcessingUnit
16 from schainpy.model.proc.jroproc_base import ProcessingUnit
17 from schainpy.model.data.jrodata import Parameters
17 from schainpy.model.data.jrodata import Parameters
18 from schainpy.model.io.jroIO_base import JRODataReader, isNumber
18 from schainpy.model.io.jroIO_base import JRODataReader, isNumber
19
19
20 FILE_HEADER_STRUCTURE = numpy.dtype([
20 FILE_HEADER_STRUCTURE = numpy.dtype([
21 ('FMN', '<u4'),
21 ('FMN', '<u4'),
22 ('nrec', '<u4'),
22 ('nrec', '<u4'),
23 ('fr_offset', '<u4'),
23 ('fr_offset', '<u4'),
24 ('id', '<u4'),
24 ('id', '<u4'),
25 ('site', 'u1', (32,))
25 ('site', 'u1', (32,))
26 ])
26 ])
27
27
28 REC_HEADER_STRUCTURE = numpy.dtype([
28 REC_HEADER_STRUCTURE = numpy.dtype([
29 ('rmn', '<u4'),
29 ('rmn', '<u4'),
30 ('rcounter', '<u4'),
30 ('rcounter', '<u4'),
31 ('nr_offset', '<u4'),
31 ('nr_offset', '<u4'),
32 ('tr_offset', '<u4'),
32 ('tr_offset', '<u4'),
33 ('time', '<u4'),
33 ('time', '<u4'),
34 ('time_msec', '<u4'),
34 ('time_msec', '<u4'),
35 ('tag', 'u1', (32,)),
35 ('tag', 'u1', (32,)),
36 ('comments', 'u1', (32,)),
36 ('comments', 'u1', (32,)),
37 ('lat', '<f4'),
37 ('lat', '<f4'),
38 ('lon', '<f4'),
38 ('lon', '<f4'),
39 ('gps_status', '<u4'),
39 ('gps_status', '<u4'),
40 ('freq', '<u4'),
40 ('freq', '<u4'),
41 ('freq0', '<u4'),
41 ('freq0', '<u4'),
42 ('nchan', '<u4'),
42 ('nchan', '<u4'),
43 ('delta_r', '<u4'),
43 ('delta_r', '<u4'),
44 ('nranges', '<u4'),
44 ('nranges', '<u4'),
45 ('r0', '<u4'),
45 ('r0', '<u4'),
46 ('prf', '<u4'),
46 ('prf', '<u4'),
47 ('ncoh', '<u4'),
47 ('ncoh', '<u4'),
48 ('npoints', '<u4'),
48 ('npoints', '<u4'),
49 ('polarization', '<i4'),
49 ('polarization', '<i4'),
50 ('rx_filter', '<u4'),
50 ('rx_filter', '<u4'),
51 ('nmodes', '<u4'),
51 ('nmodes', '<u4'),
52 ('dmode_index', '<u4'),
52 ('dmode_index', '<u4'),
53 ('dmode_rngcorr', '<u4'),
53 ('dmode_rngcorr', '<u4'),
54 ('nrxs', '<u4'),
54 ('nrxs', '<u4'),
55 ('acf_length', '<u4'),
55 ('acf_length', '<u4'),
56 ('acf_lags', '<u4'),
56 ('acf_lags', '<u4'),
57 ('sea_to_atmos', '<f4'),
57 ('sea_to_atmos', '<f4'),
58 ('sea_notch', '<u4'),
58 ('sea_notch', '<u4'),
59 ('lh_sea', '<u4'),
59 ('lh_sea', '<u4'),
60 ('hh_sea', '<u4'),
60 ('hh_sea', '<u4'),
61 ('nbins_sea', '<u4'),
61 ('nbins_sea', '<u4'),
62 ('min_snr', '<f4'),
62 ('min_snr', '<f4'),
63 ('min_cc', '<f4'),
63 ('min_cc', '<f4'),
64 ('max_time_diff', '<f4')
64 ('max_time_diff', '<f4')
65 ])
65 ])
66
66
67 DATA_STRUCTURE = numpy.dtype([
67 DATA_STRUCTURE = numpy.dtype([
68 ('range', '<u4'),
68 ('range', '<u4'),
69 ('status', '<u4'),
69 ('status', '<u4'),
70 ('zonal', '<f4'),
70 ('zonal', '<f4'),
71 ('meridional', '<f4'),
71 ('meridional', '<f4'),
72 ('vertical', '<f4'),
72 ('vertical', '<f4'),
73 ('zonal_a', '<f4'),
73 ('zonal_a', '<f4'),
74 ('meridional_a', '<f4'),
74 ('meridional_a', '<f4'),
75 ('corrected_fading', '<f4'), # seconds
75 ('corrected_fading', '<f4'), # seconds
76 ('uncorrected_fading', '<f4'), # seconds
76 ('uncorrected_fading', '<f4'), # seconds
77 ('time_diff', '<f4'),
77 ('time_diff', '<f4'),
78 ('major_axis', '<f4'),
78 ('major_axis', '<f4'),
79 ('axial_ratio', '<f4'),
79 ('axial_ratio', '<f4'),
80 ('orientation', '<f4'),
80 ('orientation', '<f4'),
81 ('sea_power', '<u4'),
81 ('sea_power', '<u4'),
82 ('sea_algorithm', '<u4')
82 ('sea_algorithm', '<u4')
83 ])
83 ])
84
84
85
85 class BLTRParamReader(JRODataReader, ProcessingUnit):
86 class BLTRParamReader(JRODataReader, ProcessingUnit):
86 '''
87 '''
87 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR from *.sswma files
88 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR from *.sswma files
88 '''
89 '''
89
90
90 ext = '.sswma'
91 ext = '.sswma'
91
92
92 def __init__(self, **kwargs):
93 def __init__(self, **kwargs):
93
94
94 ProcessingUnit.__init__(self , **kwargs)
95 ProcessingUnit.__init__(self, **kwargs)
95
96
96 self.dataOut = Parameters()
97 self.dataOut = Parameters()
97 self.counter_records = 0
98 self.counter_records = 0
98 self.flagNoMoreFiles = 0
99 self.flagNoMoreFiles = 0
99 self.isConfig = False
100 self.isConfig = False
100 self.filename = None
101 self.filename = None
101
102
102 def setup(self,
103 def setup(self,
103 path=None,
104 path=None,
104 startDate=None,
105 startDate=None,
105 endDate=None,
106 endDate=None,
106 ext=None,
107 ext=None,
107 startTime=datetime.time(0, 0, 0),
108 startTime=datetime.time(0, 0, 0),
108 endTime=datetime.time(23, 59, 59),
109 endTime=datetime.time(23, 59, 59),
109 timezone=0,
110 timezone=0,
110 status_value=0,
111 status_value=0, **kwargs):
111 **kwargs):
112
112
113 self.path = path
113 self.path = path
114 self.startTime = startTime
114 self.startTime = startTime
115 self.endTime = endTime
115 self.endTime = endTime
116 self.status_value = status_value
116 self.status_value = status_value
117
117
118 if self.path is None:
118 if self.path is None:
119 raise ValueError, "The path is not valid"
119 raise ValueError, "The path is not valid"
120
120
121 if ext is None:
121 if ext is None:
122 ext = self.ext
122 ext = self.ext
123
123
124 self.search_files(self.path, startDate, endDate, ext)
124 self.search_files(self.path, startDate, endDate, ext)
125 self.timezone = timezone
125 self.timezone = timezone
126 self.fileIndex = 0
126 self.fileIndex = 0
127
127
128 if not self.fileList:
128 if not self.fileList:
129 raise Warning, "There is no files matching these date in the folder: %s. \n Check 'startDate' and 'endDate' "%(path)
129 raise Warning, "There is no files matching these date in the folder: %s. \n Check 'startDate' and 'endDate' " % (
130 path)
130
131
131 self.setNextFile()
132 self.setNextFile()
132
133
133 def search_files(self, path, startDate, endDate, ext):
134 def search_files(self, path, startDate, endDate, ext):
134 '''
135 '''
135 Searching for BLTR rawdata file in path
136 Searching for BLTR rawdata file in path
136 Creating a list of file to proces included in [startDate,endDate]
137 Creating a list of file to proces included in [startDate,endDate]
137
138
138 Input:
139 Input:
139 path - Path to find BLTR rawdata files
140 path - Path to find BLTR rawdata files
140 startDate - Select file from this date
141 startDate - Select file from this date
141 enDate - Select file until this date
142 enDate - Select file until this date
142 ext - Extension of the file to read
143 ext - Extension of the file to read
143
144 '''
145
144
146 print 'Searching file in %s ' % (path)
145 '''
147 foldercounter = 0
146
147 print 'Searching file in %s ' % (path)
148 foldercounter = 0
148 fileList0 = glob.glob1(path, "*%s" % ext)
149 fileList0 = glob.glob1(path, "*%s" % ext)
149 fileList0.sort()
150 fileList0.sort()
150
151
151 self.fileList = []
152 self.fileList = []
152 self.dateFileList = []
153 self.dateFileList = []
153
154
154 for thisFile in fileList0:
155 for thisFile in fileList0:
155 year = thisFile[-14:-10]
156 year = thisFile[-14:-10]
156 if not isNumber(year):
157 if not isNumber(year):
157 continue
158 continue
158
159
159 month = thisFile[-10:-8]
160 month = thisFile[-10:-8]
160 if not isNumber(month):
161 if not isNumber(month):
161 continue
162 continue
162
163
163 day = thisFile[-8:-6]
164 day = thisFile[-8:-6]
164 if not isNumber(day):
165 if not isNumber(day):
165 continue
166 continue
166
167
167 year, month, day = int(year), int(month), int(day)
168 year, month, day = int(year), int(month), int(day)
168 dateFile = datetime.date(year, month, day)
169 dateFile = datetime.date(year, month, day)
169
170
170 if (startDate > dateFile) or (endDate < dateFile):
171 if (startDate > dateFile) or (endDate < dateFile):
171 continue
172 continue
172
173
173 self.fileList.append(thisFile)
174 self.fileList.append(thisFile)
174 self.dateFileList.append(dateFile)
175 self.dateFileList.append(dateFile)
175
176
176 return
177 return
177
178
178 def setNextFile(self):
179 def setNextFile(self):
179
180
180 file_id = self.fileIndex
181 file_id = self.fileIndex
181
182
182 if file_id == len(self.fileList):
183 if file_id == len(self.fileList):
183 print '\nNo more files in the folder'
184 print '\nNo more files in the folder'
184 print 'Total number of file(s) read : {}'.format(self.fileIndex + 1)
185 print 'Total number of file(s) read : {}'.format(self.fileIndex + 1)
185 self.flagNoMoreFiles = 1
186 self.flagNoMoreFiles = 1
186 return 0
187 return 0
187
188
188 print '\n[Setting file] (%s) ...' % self.fileList[file_id]
189 print '\n[Setting file] (%s) ...' % self.fileList[file_id]
189 filename = os.path.join(self.path, self.fileList[file_id])
190 filename = os.path.join(self.path, self.fileList[file_id])
190
191
191 dirname, name = os.path.split(filename)
192 dirname, name = os.path.split(filename)
192 self.siteFile = name.split('.')[0] # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
193 # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
194 self.siteFile = name.split('.')[0]
193 if self.filename is not None:
195 if self.filename is not None:
194 self.fp.close()
196 self.fp.close()
195 self.filename = filename
197 self.filename = filename
196 self.fp = open(self.filename, 'rb')
198 self.fp = open(self.filename, 'rb')
197 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
199 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
198 self.nrecords = self.header_file['nrec'][0]
200 self.nrecords = self.header_file['nrec'][0]
199 self.sizeOfFile = os.path.getsize(self.filename)
201 self.sizeOfFile = os.path.getsize(self.filename)
200 self.counter_records = 0
202 self.counter_records = 0
201 self.flagIsNewFile = 0
203 self.flagIsNewFile = 0
202 self.fileIndex += 1
204 self.fileIndex += 1
203
205
204 return 1
206 return 1
205
207
206 def readNextBlock(self):
208 def readNextBlock(self):
207
209
208 while True:
210 while True:
209 if self.counter_records == self.nrecords:
211 if self.counter_records == self.nrecords:
210 self.flagIsNewFile = 1
212 self.flagIsNewFile = 1
211 if not self.setNextFile():
213 if not self.setNextFile():
212 return 0
214 return 0
213
215
214 self.readBlock()
216 self.readBlock()
215
217
216 if (self.datatime.time() < self.startTime) or (self.datatime.time() > self.endTime):
218 if (self.datatime.time() < self.startTime) or (self.datatime.time() > self.endTime):
217 print "[Reading] Record No. %d/%d -> %s [Skipping]" %(
219 print "[Reading] Record No. %d/%d -> %s [Skipping]" % (
218 self.counter_records,
220 self.counter_records,
219 self.nrecords,
221 self.nrecords,
220 self.datatime.ctime())
222 self.datatime.ctime())
221 continue
223 continue
222 break
224 break
223
225
224 print "[Reading] Record No. %d/%d -> %s" %(
226 print "[Reading] Record No. %d/%d -> %s" % (
225 self.counter_records,
227 self.counter_records,
226 self.nrecords,
228 self.nrecords,
227 self.datatime.ctime())
229 self.datatime.ctime())
228
230
229 return 1
231 return 1
230
232
231 def readBlock(self):
233 def readBlock(self):
232
234
233 pointer = self.fp.tell()
235 pointer = self.fp.tell()
234 header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
236 header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
235 self.nchannels = header_rec['nchan'][0]/2
237 self.nchannels = header_rec['nchan'][0] / 2
236 self.kchan = header_rec['nrxs'][0]
238 self.kchan = header_rec['nrxs'][0]
237 self.nmodes = header_rec['nmodes'][0]
239 self.nmodes = header_rec['nmodes'][0]
238 self.nranges = header_rec['nranges'][0]
240 self.nranges = header_rec['nranges'][0]
239 self.fp.seek(pointer)
241 self.fp.seek(pointer)
240 self.height = numpy.empty((self.nmodes, self.nranges))
242 self.height = numpy.empty((self.nmodes, self.nranges))
241 self.snr = numpy.empty((self.nmodes, self.nchannels, self.nranges))
243 self.snr = numpy.empty((self.nmodes, self.nchannels, self.nranges))
242 self.buffer = numpy.empty((self.nmodes, 3, self.nranges))
244 self.buffer = numpy.empty((self.nmodes, 3, self.nranges))
243
245
244 for mode in range(self.nmodes):
246 for mode in range(self.nmodes):
245 self.readHeader()
247 self.readHeader()
246 data = self.readData()
248 data = self.readData()
247 self.height[mode] = (data[0] - self.correction) / 1000.
249 self.height[mode] = (data[0] - self.correction) / 1000.
248 self.buffer[mode] = data[1]
250 self.buffer[mode] = data[1]
249 self.snr[mode] = data[2]
251 self.snr[mode] = data[2]
250
252
251 self.counter_records = self.counter_records + self.nmodes
253 self.counter_records = self.counter_records + self.nmodes
252
254
253 return
255 return
254
256
255 def readHeader(self):
257 def readHeader(self):
256 '''
258 '''
257 RecordHeader of BLTR rawdata file
259 RecordHeader of BLTR rawdata file
258 '''
260 '''
259
261
260 header_structure = numpy.dtype(
262 header_structure = numpy.dtype(
261 REC_HEADER_STRUCTURE.descr + [
263 REC_HEADER_STRUCTURE.descr + [
262 ('antenna_coord', 'f4', (2, self.nchannels)),
264 ('antenna_coord', 'f4', (2, self.nchannels)),
263 ('rx_gains', 'u4', (self.nchannels,)),
265 ('rx_gains', 'u4', (self.nchannels,)),
264 ('rx_analysis', 'u4', (self.nchannels,))
266 ('rx_analysis', 'u4', (self.nchannels,))
265 ]
267 ]
266 )
268 )
267
269
268 self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
270 self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
269 self.lat = self.header_rec['lat'][0]
271 self.lat = self.header_rec['lat'][0]
270 self.lon = self.header_rec['lon'][0]
272 self.lon = self.header_rec['lon'][0]
271 self.delta = self.header_rec['delta_r'][0]
273 self.delta = self.header_rec['delta_r'][0]
272 self.correction = self.header_rec['dmode_rngcorr'][0]
274 self.correction = self.header_rec['dmode_rngcorr'][0]
273 self.imode = self.header_rec['dmode_index'][0]
275 self.imode = self.header_rec['dmode_index'][0]
274 self.antenna = self.header_rec['antenna_coord']
276 self.antenna = self.header_rec['antenna_coord']
275 self.rx_gains = self.header_rec['rx_gains']
277 self.rx_gains = self.header_rec['rx_gains']
276 self.time = self.header_rec['time'][0]
278 self.time = self.header_rec['time'][0]
277 tseconds = self.header_rec['time'][0]
279 tseconds = self.header_rec['time'][0]
278 local_t1 = time.localtime(tseconds)
280 local_t1 = time.localtime(tseconds)
279 self.year = local_t1.tm_year
281 self.year = local_t1.tm_year
280 self.month = local_t1.tm_mon
282 self.month = local_t1.tm_mon
281 self.day = local_t1.tm_mday
283 self.day = local_t1.tm_mday
282 self.t = datetime.datetime(self.year, self.month, self.day)
284 self.t = datetime.datetime(self.year, self.month, self.day)
283 self.datatime = datetime.datetime.utcfromtimestamp(self.time)
285 self.datatime = datetime.datetime.utcfromtimestamp(self.time)
284
286
285 def readData(self):
287 def readData(self):
286 '''
288 '''
287 Reading and filtering data block record of BLTR rawdata file, filtering is according to status_value.
289 Reading and filtering data block record of BLTR rawdata file, filtering is according to status_value.
288
290
289 Input:
291 Input:
290 status_value - Array data is set to NAN for values that are not equal to status_value
292 status_value - Array data is set to NAN for values that are not equal to status_value
291
293
292 '''
294 '''
293
295
294 data_structure = numpy.dtype(
296 data_structure = numpy.dtype(
295 DATA_STRUCTURE.descr + [
297 DATA_STRUCTURE.descr + [
296 ('rx_saturation', 'u4', (self.nchannels,)),
298 ('rx_saturation', 'u4', (self.nchannels,)),
297 ('chan_offset', 'u4', (2 * self.nchannels,)),
299 ('chan_offset', 'u4', (2 * self.nchannels,)),
298 ('rx_amp', 'u4', (self.nchannels,)),
300 ('rx_amp', 'u4', (self.nchannels,)),
299 ('rx_snr', 'f4', (self.nchannels,)),
301 ('rx_snr', 'f4', (self.nchannels,)),
300 ('cross_snr', 'f4', (self.kchan,)),
302 ('cross_snr', 'f4', (self.kchan,)),
301 ('sea_power_relative', 'f4', (self.kchan,))]
303 ('sea_power_relative', 'f4', (self.kchan,))]
302 )
304 )
303
305
304 data = numpy.fromfile(self.fp, data_structure, self.nranges)
306 data = numpy.fromfile(self.fp, data_structure, self.nranges)
305
307
306 height = data['range']
308 height = data['range']
307 winds = numpy.array((data['zonal'], data['meridional'], data['vertical']))
309 winds = numpy.array(
310 (data['zonal'], data['meridional'], data['vertical']))
308 snr = data['rx_snr'].T
311 snr = data['rx_snr'].T
309
312
310 winds[numpy.where(winds == -9999.)] = numpy.nan
313 winds[numpy.where(winds == -9999.)] = numpy.nan
311 winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
314 winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
312 snr[numpy.where(snr == -9999.)] = numpy.nan
315 snr[numpy.where(snr == -9999.)] = numpy.nan
313 snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
316 snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
314 snr = numpy.power(10, snr / 10)
317 snr = numpy.power(10, snr / 10)
315
318
316 return height, winds, snr
319 return height, winds, snr
317
320
318 def set_output(self):
321 def set_output(self):
319 '''
322 '''
320 Storing data from databuffer to dataOut object
323 Storing data from databuffer to dataOut object
321 '''
324 '''
322
325
323 self.dataOut.data_SNR = self.snr
326 self.dataOut.data_SNR = self.snr
324 self.dataOut.height = self.height
327 self.dataOut.height = self.height
325 self.dataOut.data_output = self.buffer
328 self.dataOut.data_output = self.buffer
326 self.dataOut.utctimeInit = self.time
329 self.dataOut.utctimeInit = self.time
327 self.dataOut.utctime = self.dataOut.utctimeInit
330 self.dataOut.utctime = self.dataOut.utctimeInit
328 self.dataOut.useLocalTime = False
331 self.dataOut.useLocalTime = False
329 self.dataOut.paramInterval = 157
332 self.dataOut.paramInterval = 157
330 self.dataOut.timezone = self.timezone
333 self.dataOut.timezone = self.timezone
331 self.dataOut.site = self.siteFile
334 self.dataOut.site = self.siteFile
332 self.dataOut.nrecords = self.nrecords/self.nmodes
335 self.dataOut.nrecords = self.nrecords / self.nmodes
333 self.dataOut.sizeOfFile = self.sizeOfFile
336 self.dataOut.sizeOfFile = self.sizeOfFile
334 self.dataOut.lat = self.lat
337 self.dataOut.lat = self.lat
335 self.dataOut.lon = self.lon
338 self.dataOut.lon = self.lon
336 self.dataOut.channelList = range(self.nchannels)
339 self.dataOut.channelList = range(self.nchannels)
337 self.dataOut.kchan = self.kchan
340 self.dataOut.kchan = self.kchan
338 # self.dataOut.nHeights = self.nranges
341 # self.dataOut.nHeights = self.nranges
339 self.dataOut.delta = self.delta
342 self.dataOut.delta = self.delta
340 self.dataOut.correction = self.correction
343 self.dataOut.correction = self.correction
341 self.dataOut.nmodes = self.nmodes
344 self.dataOut.nmodes = self.nmodes
342 self.dataOut.imode = self.imode
345 self.dataOut.imode = self.imode
343 self.dataOut.antenna = self.antenna
346 self.dataOut.antenna = self.antenna
344 self.dataOut.rx_gains = self.rx_gains
347 self.dataOut.rx_gains = self.rx_gains
345 self.dataOut.flagNoData = False
348 self.dataOut.flagNoData = False
346
349
347 def getData(self):
350 def getData(self):
348 '''
351 '''
349 Storing data from databuffer to dataOut object
352 Storing data from databuffer to dataOut object
350 '''
353 '''
351 if self.flagNoMoreFiles:
354 if self.flagNoMoreFiles:
352 self.dataOut.flagNoData = True
355 self.dataOut.flagNoData = True
353 print 'No file left to process'
356 print 'No file left to process'
354 return 0
357 return 0
355
358
356 if not self.readNextBlock():
359 if not self.readNextBlock():
357 self.dataOut.flagNoData = True
360 self.dataOut.flagNoData = True
358 return 0
361 return 0
359
362
360 self.set_output()
363 self.set_output()
361
364
362 return 1
365 return 1
General Comments 0
You need to be logged in to leave comments. Login now