##// END OF EJS Templates
schain-jc
RSS
formatting
José Chávez -
r1083:e4f5cab49468
parent child
Show More
Side by Side Unified
Context file:
r1083:e4f5cab49468 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,362 +1,365
1 1 '''
2 2 Created on Nov 9, 2016
3 3
4 4 @author: roj- LouVD
5 5 '''
6 6
7 7
8 8 import os
9 9 import sys
10 10 import time
11 11 import glob
12 12 import datetime
13 13
14 14 import numpy
15 15
16 16 from schainpy.model.proc.jroproc_base import ProcessingUnit
17 17 from schainpy.model.data.jrodata import Parameters
18 18 from schainpy.model.io.jroIO_base import JRODataReader, isNumber
19 19
20 20 FILE_HEADER_STRUCTURE = numpy.dtype([
21 21 ('FMN', '<u4'),
22 22 ('nrec', '<u4'),
23 23 ('fr_offset', '<u4'),
24 24 ('id', '<u4'),
25 25 ('site', 'u1', (32,))
26 26 ])
27 27
28 28 REC_HEADER_STRUCTURE = numpy.dtype([
29 29 ('rmn', '<u4'),
30 30 ('rcounter', '<u4'),
31 31 ('nr_offset', '<u4'),
32 32 ('tr_offset', '<u4'),
33 33 ('time', '<u4'),
34 34 ('time_msec', '<u4'),
35 35 ('tag', 'u1', (32,)),
36 36 ('comments', 'u1', (32,)),
37 37 ('lat', '<f4'),
38 38 ('lon', '<f4'),
39 39 ('gps_status', '<u4'),
40 40 ('freq', '<u4'),
41 41 ('freq0', '<u4'),
42 42 ('nchan', '<u4'),
43 43 ('delta_r', '<u4'),
44 44 ('nranges', '<u4'),
45 45 ('r0', '<u4'),
46 46 ('prf', '<u4'),
47 47 ('ncoh', '<u4'),
48 48 ('npoints', '<u4'),
49 49 ('polarization', '<i4'),
50 50 ('rx_filter', '<u4'),
51 51 ('nmodes', '<u4'),
52 52 ('dmode_index', '<u4'),
53 53 ('dmode_rngcorr', '<u4'),
54 54 ('nrxs', '<u4'),
55 55 ('acf_length', '<u4'),
56 56 ('acf_lags', '<u4'),
57 57 ('sea_to_atmos', '<f4'),
58 58 ('sea_notch', '<u4'),
59 59 ('lh_sea', '<u4'),
60 60 ('hh_sea', '<u4'),
61 61 ('nbins_sea', '<u4'),
62 62 ('min_snr', '<f4'),
63 63 ('min_cc', '<f4'),
64 64 ('max_time_diff', '<f4')
65 65 ])
66 66
67 67 DATA_STRUCTURE = numpy.dtype([
68 68 ('range', '<u4'),
69 69 ('status', '<u4'),
70 70 ('zonal', '<f4'),
71 71 ('meridional', '<f4'),
72 72 ('vertical', '<f4'),
73 73 ('zonal_a', '<f4'),
74 74 ('meridional_a', '<f4'),
75 75 ('corrected_fading', '<f4'), # seconds
76 76 ('uncorrected_fading', '<f4'), # seconds
77 77 ('time_diff', '<f4'),
78 78 ('major_axis', '<f4'),
79 79 ('axial_ratio', '<f4'),
80 80 ('orientation', '<f4'),
81 81 ('sea_power', '<u4'),
82 82 ('sea_algorithm', '<u4')
83 83 ])
84 84
85
85 86 class BLTRParamReader(JRODataReader, ProcessingUnit):
86 87 '''
87 88 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR from *.sswma files
88 89 '''
89 90
90 91 ext = '.sswma'
91 92
92 93 def __init__(self, **kwargs):
93 94
94 95 ProcessingUnit.__init__(self , **kwargs)
95 96
96 97 self.dataOut = Parameters()
97 98 self.counter_records = 0
98 99 self.flagNoMoreFiles = 0
99 100 self.isConfig = False
100 101 self.filename = None
101 102
102 103 def setup(self,
103 104 path=None,
104 105 startDate=None,
105 106 endDate=None,
106 107 ext=None,
107 108 startTime=datetime.time(0, 0, 0),
108 109 endTime=datetime.time(23, 59, 59),
109 110 timezone=0,
110 status_value=0,
111 **kwargs):
111 status_value=0, **kwargs):
112 112
113 113 self.path = path
114 114 self.startTime = startTime
115 115 self.endTime = endTime
116 116 self.status_value = status_value
117 117
118 118 if self.path is None:
119 119 raise ValueError, "The path is not valid"
120 120
121 121 if ext is None:
122 122 ext = self.ext
123 123
124 124 self.search_files(self.path, startDate, endDate, ext)
125 125 self.timezone = timezone
126 126 self.fileIndex = 0
127 127
128 128 if not self.fileList:
129 raise Warning, "There is no files matching these date in the folder: %s. \n Check 'startDate' and 'endDate' "%(path)
129 raise Warning, "There is no files matching these date in the folder: %s. \n Check 'startDate' and 'endDate' " % (
130 path)
130 131
131 132 self.setNextFile()
132 133
133 134 def search_files(self, path, startDate, endDate, ext):
134 135 '''
135 136 Searching for BLTR rawdata file in path
136 137 Creating a list of file to proces included in [startDate,endDate]
137 138
138 139 Input:
139 140 path - Path to find BLTR rawdata files
140 141 startDate - Select file from this date
141 142 enDate - Select file until this date
142 143 ext - Extension of the file to read
143 144
144 145 '''
145 146
146 147 print 'Searching file in %s ' % (path)
147 148 foldercounter = 0
148 149 fileList0 = glob.glob1(path, "*%s" % ext)
149 150 fileList0.sort()
150 151
151 152 self.fileList = []
152 153 self.dateFileList = []
153 154
154 155 for thisFile in fileList0:
155 156 year = thisFile[-14:-10]
156 157 if not isNumber(year):
157 158 continue
158 159
159 160 month = thisFile[-10:-8]
160 161 if not isNumber(month):
161 162 continue
162 163
163 164 day = thisFile[-8:-6]
164 165 if not isNumber(day):
165 166 continue
166 167
167 168 year, month, day = int(year), int(month), int(day)
168 169 dateFile = datetime.date(year, month, day)
169 170
170 171 if (startDate > dateFile) or (endDate < dateFile):
171 172 continue
172 173
173 174 self.fileList.append(thisFile)
174 175 self.dateFileList.append(dateFile)
175 176
176 177 return
177 178
178 179 def setNextFile(self):
179 180
180 181 file_id = self.fileIndex
181 182
182 183 if file_id == len(self.fileList):
183 184 print '\nNo more files in the folder'
184 185 print 'Total number of file(s) read : {}'.format(self.fileIndex + 1)
185 186 self.flagNoMoreFiles = 1
186 187 return 0
187 188
188 189 print '\n[Setting file] (%s) ...' % self.fileList[file_id]
189 190 filename = os.path.join(self.path, self.fileList[file_id])
190 191
191 192 dirname, name = os.path.split(filename)
192 self.siteFile = name.split('.')[0] # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
193 # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
194 self.siteFile = name.split('.')[0]
193 195 if self.filename is not None:
194 196 self.fp.close()
195 197 self.filename = filename
196 198 self.fp = open(self.filename, 'rb')
197 199 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
198 200 self.nrecords = self.header_file['nrec'][0]
199 201 self.sizeOfFile = os.path.getsize(self.filename)
200 202 self.counter_records = 0
201 203 self.flagIsNewFile = 0
202 204 self.fileIndex += 1
203 205
204 206 return 1
205 207
206 208 def readNextBlock(self):
207 209
208 210 while True:
209 211 if self.counter_records == self.nrecords:
210 212 self.flagIsNewFile = 1
211 213 if not self.setNextFile():
212 214 return 0
213 215
214 216 self.readBlock()
215 217
216 218 if (self.datatime.time() < self.startTime) or (self.datatime.time() > self.endTime):
217 219 print "[Reading] Record No. %d/%d -> %s [Skipping]" %(
218 220 self.counter_records,
219 221 self.nrecords,
220 222 self.datatime.ctime())
221 223 continue
222 224 break
223 225
224 226 print "[Reading] Record No. %d/%d -> %s" %(
225 227 self.counter_records,
226 228 self.nrecords,
227 229 self.datatime.ctime())
228 230
229 231 return 1
230 232
231 233 def readBlock(self):
232 234
233 235 pointer = self.fp.tell()
234 236 header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
235 237 self.nchannels = header_rec['nchan'][0]/2
236 238 self.kchan = header_rec['nrxs'][0]
237 239 self.nmodes = header_rec['nmodes'][0]
238 240 self.nranges = header_rec['nranges'][0]
239 241 self.fp.seek(pointer)
240 242 self.height = numpy.empty((self.nmodes, self.nranges))
241 243 self.snr = numpy.empty((self.nmodes, self.nchannels, self.nranges))
242 244 self.buffer = numpy.empty((self.nmodes, 3, self.nranges))
243 245
244 246 for mode in range(self.nmodes):
245 247 self.readHeader()
246 248 data = self.readData()
247 249 self.height[mode] = (data[0] - self.correction) / 1000.
248 250 self.buffer[mode] = data[1]
249 251 self.snr[mode] = data[2]
250 252
251 253 self.counter_records = self.counter_records + self.nmodes
252 254
253 255 return
254 256
255 257 def readHeader(self):
256 258 '''
257 259 RecordHeader of BLTR rawdata file
258 260 '''
259 261
260 262 header_structure = numpy.dtype(
261 263 REC_HEADER_STRUCTURE.descr + [
262 264 ('antenna_coord', 'f4', (2, self.nchannels)),
263 265 ('rx_gains', 'u4', (self.nchannels,)),
264 266 ('rx_analysis', 'u4', (self.nchannels,))
265 267 ]
266 268 )
267 269
268 270 self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
269 271 self.lat = self.header_rec['lat'][0]
270 272 self.lon = self.header_rec['lon'][0]
271 273 self.delta = self.header_rec['delta_r'][0]
272 274 self.correction = self.header_rec['dmode_rngcorr'][0]
273 275 self.imode = self.header_rec['dmode_index'][0]
274 276 self.antenna = self.header_rec['antenna_coord']
275 277 self.rx_gains = self.header_rec['rx_gains']
276 278 self.time = self.header_rec['time'][0]
277 279 tseconds = self.header_rec['time'][0]
278 280 local_t1 = time.localtime(tseconds)
279 281 self.year = local_t1.tm_year
280 282 self.month = local_t1.tm_mon
281 283 self.day = local_t1.tm_mday
282 284 self.t = datetime.datetime(self.year, self.month, self.day)
283 285 self.datatime = datetime.datetime.utcfromtimestamp(self.time)
284 286
285 287 def readData(self):
286 288 '''
287 289 Reading and filtering data block record of BLTR rawdata file, filtering is according to status_value.
288 290
289 291 Input:
290 292 status_value - Array data is set to NAN for values that are not equal to status_value
291 293
292 294 '''
293 295
294 296 data_structure = numpy.dtype(
295 297 DATA_STRUCTURE.descr + [
296 298 ('rx_saturation', 'u4', (self.nchannels,)),
297 299 ('chan_offset', 'u4', (2 * self.nchannels,)),
298 300 ('rx_amp', 'u4', (self.nchannels,)),
299 301 ('rx_snr', 'f4', (self.nchannels,)),
300 302 ('cross_snr', 'f4', (self.kchan,)),
301 303 ('sea_power_relative', 'f4', (self.kchan,))]
302 304 )
303 305
304 306 data = numpy.fromfile(self.fp, data_structure, self.nranges)
305 307
306 308 height = data['range']
307 winds = numpy.array((data['zonal'], data['meridional'], data['vertical']))
309 winds = numpy.array(
310 (data['zonal'], data['meridional'], data['vertical']))
308 311 snr = data['rx_snr'].T
309 312
310 313 winds[numpy.where(winds == -9999.)] = numpy.nan
311 314 winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
312 315 snr[numpy.where(snr == -9999.)] = numpy.nan
313 316 snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
314 317 snr = numpy.power(10, snr / 10)
315 318
316 319 return height, winds, snr
317 320
318 321 def set_output(self):
319 322 '''
320 323 Storing data from databuffer to dataOut object
321 324 '''
322 325
323 326 self.dataOut.data_SNR = self.snr
324 327 self.dataOut.height = self.height
325 328 self.dataOut.data_output = self.buffer
326 329 self.dataOut.utctimeInit = self.time
327 330 self.dataOut.utctime = self.dataOut.utctimeInit
328 331 self.dataOut.useLocalTime = False
329 332 self.dataOut.paramInterval = 157
330 333 self.dataOut.timezone = self.timezone
331 334 self.dataOut.site = self.siteFile
332 335 self.dataOut.nrecords = self.nrecords/self.nmodes
333 336 self.dataOut.sizeOfFile = self.sizeOfFile
334 337 self.dataOut.lat = self.lat
335 338 self.dataOut.lon = self.lon
336 339 self.dataOut.channelList = range(self.nchannels)
337 340 self.dataOut.kchan = self.kchan
338 341 # self.dataOut.nHeights = self.nranges
339 342 self.dataOut.delta = self.delta
340 343 self.dataOut.correction = self.correction
341 344 self.dataOut.nmodes = self.nmodes
342 345 self.dataOut.imode = self.imode
343 346 self.dataOut.antenna = self.antenna
344 347 self.dataOut.rx_gains = self.rx_gains
345 348 self.dataOut.flagNoData = False
346 349
347 350 def getData(self):
348 351 '''
349 352 Storing data from databuffer to dataOut object
350 353 '''
351 354 if self.flagNoMoreFiles:
352 355 self.dataOut.flagNoData = True
353 356 print 'No file left to process'
354 357 return 0
355 358
356 359 if not self.readNextBlock():
357 360 self.dataOut.flagNoData = True
358 361 return 0
359 362
360 363 self.set_output()
361 364
362 365 return 1
General Comments 0
You need to be logged in to leave comments. Login now