##// END OF EJS Templates
schain-jc
RSS
Fix reading files in online mode
Juan C. Espinoza -
r1217:f42f79a936d7
parent child
Show More
Side by Side Unified
Context file:
r1217:f42f79a936d7 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,409 +1,406
1 1 '''
2 2 Created on Nov 9, 2016
3 3
4 4 @author: roj- LouVD
5 5 '''
6 6
7 7
8 8 import os
9 9 import sys
10 10 import time
11 11 import glob
12 12 import datetime
13 13
14 14 import numpy
15 15
16 16 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator
17 17 from schainpy.model.data.jrodata import Parameters
18 18 from schainpy.model.io.jroIO_base import JRODataReader, isNumber
19 19 from schainpy.utils import log
20 20
21 21 FILE_HEADER_STRUCTURE = numpy.dtype([
22 22 ('FMN', '<u4'),
23 23 ('nrec', '<u4'),
24 24 ('fr_offset', '<u4'),
25 25 ('id', '<u4'),
26 26 ('site', 'u1', (32,))
27 27 ])
28 28
29 29 REC_HEADER_STRUCTURE = numpy.dtype([
30 30 ('rmn', '<u4'),
31 31 ('rcounter', '<u4'),
32 32 ('nr_offset', '<u4'),
33 33 ('tr_offset', '<u4'),
34 34 ('time', '<u4'),
35 35 ('time_msec', '<u4'),
36 36 ('tag', 'u1', (32,)),
37 37 ('comments', 'u1', (32,)),
38 38 ('lat', '<f4'),
39 39 ('lon', '<f4'),
40 40 ('gps_status', '<u4'),
41 41 ('freq', '<u4'),
42 42 ('freq0', '<u4'),
43 43 ('nchan', '<u4'),
44 44 ('delta_r', '<u4'),
45 45 ('nranges', '<u4'),
46 46 ('r0', '<u4'),
47 47 ('prf', '<u4'),
48 48 ('ncoh', '<u4'),
49 49 ('npoints', '<u4'),
50 50 ('polarization', '<i4'),
51 51 ('rx_filter', '<u4'),
52 52 ('nmodes', '<u4'),
53 53 ('dmode_index', '<u4'),
54 54 ('dmode_rngcorr', '<u4'),
55 55 ('nrxs', '<u4'),
56 56 ('acf_length', '<u4'),
57 57 ('acf_lags', '<u4'),
58 58 ('sea_to_atmos', '<f4'),
59 59 ('sea_notch', '<u4'),
60 60 ('lh_sea', '<u4'),
61 61 ('hh_sea', '<u4'),
62 62 ('nbins_sea', '<u4'),
63 63 ('min_snr', '<f4'),
64 64 ('min_cc', '<f4'),
65 65 ('max_time_diff', '<f4')
66 66 ])
67 67
68 68 DATA_STRUCTURE = numpy.dtype([
69 69 ('range', '<u4'),
70 70 ('status', '<u4'),
71 71 ('zonal', '<f4'),
72 72 ('meridional', '<f4'),
73 73 ('vertical', '<f4'),
74 74 ('zonal_a', '<f4'),
75 75 ('meridional_a', '<f4'),
76 76 ('corrected_fading', '<f4'), # seconds
77 77 ('uncorrected_fading', '<f4'), # seconds
78 78 ('time_diff', '<f4'),
79 79 ('major_axis', '<f4'),
80 80 ('axial_ratio', '<f4'),
81 81 ('orientation', '<f4'),
82 82 ('sea_power', '<u4'),
83 83 ('sea_algorithm', '<u4')
84 84 ])
85 85
86 86 @MPDecorator
87 87 class BLTRParamReader(JRODataReader, ProcessingUnit):
88 88 '''
89 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR from *.sswma files
89 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR
90 from *.sswma files
90 91 '''
91 92
92 93 ext = '.sswma'
93 94
94 95 def __init__(self):
95 96
96 97 ProcessingUnit.__init__(self)
97 98
98 99 self.dataOut = Parameters()
99 100 self.counter_records = 0
100 101 self.flagNoMoreFiles = 0
101 102 self.isConfig = False
102 103 self.filename = None
103 104
104 105 def setup(self,
105 106 path=None,
106 107 startDate=None,
107 108 endDate=None,
108 109 ext=None,
109 110 startTime=datetime.time(0, 0, 0),
110 111 endTime=datetime.time(23, 59, 59),
111 112 timezone=0,
112 113 status_value=0,
113 114 **kwargs):
114 115 self.path = path
115 116 self.startDate = startDate
116 117 self.endDate = endDate
117 118 self.startTime = startTime
118 119 self.endTime = endTime
119 120 self.status_value = status_value
120 121 self.datatime = datetime.datetime(1900,1,1)
121 122 self.delay = kwargs.get('delay', 10)
122 123 self.online = kwargs.get('online', False)
123 124 self.nTries = kwargs.get('nTries', 3)
124 125
125 126 if self.path is None:
126 127 raise ValueError("The path is not valid")
127 128
128 129 if ext is None:
129 130 ext = self.ext
130 131
131 132 self.fileList = self.search_files(self.path, startDate, endDate, ext)
132 133 self.timezone = timezone
133 134 self.fileIndex = 0
134 135
135 136 if not self.fileList:
136 137 raise Warning("There is no files matching these date in the folder: %s. \n Check 'startDate' and 'endDate' " % (
137 138 path))
138 139
139 140 self.setNextFile()
140 141
141 142 def search_last_file(self):
142 143 '''
143 144 Get last file and add it to the list
144 145 '''
145 146
146 147 for n in range(self.nTries):
147 if n > 0:
148 148 log.warning(
149 "Waiting %0.2f sec for the next file, try %03d ..." % (self.delay, self.nTries + 1),
149 "Waiting %0.2f seconds for the next file, try %03d ..." % (self.delay, n+1),
150 150 self.name
151 151 )
152 152 time.sleep(self.delay)
153 153 file_list = os.listdir(self.path)
154 154 file_list.sort()
155 155 if file_list:
156 156 if self.filename:
157 157 if file_list[-1] not in self.filename:
158 158 return file_list[-1]
159 159 else:
160 160 continue
161 161 return file_list[-1]
162 162 return 0
163 163
164 164 def search_files(self, path, startDate, endDate, ext):
165 165 '''
166 166 Searching for BLTR rawdata file in path
167 167 Creating a list of file to proces included in [startDate,endDate]
168 168
169 169 Input:
170 170 path - Path to find BLTR rawdata files
171 171 startDate - Select file from this date
172 172 enDate - Select file until this date
173 173 ext - Extension of the file to read
174 174 '''
175 175
176 176 log.success('Searching files in {} '.format(path), 'BLTRParamReader')
177 177 foldercounter = 0
178 178 fileList0 = glob.glob1(path, "*%s" % ext)
179 179 fileList0.sort()
180 180
181 #self.fileList = []
182 #self.dateFileList = []
183
184 181 for thisFile in fileList0:
185 182 year = thisFile[-14:-10]
186 183 if not isNumber(year):
187 184 continue
188 185
189 186 month = thisFile[-10:-8]
190 187 if not isNumber(month):
191 188 continue
192 189
193 190 day = thisFile[-8:-6]
194 191 if not isNumber(day):
195 192 continue
196 193
197 194 year, month, day = int(year), int(month), int(day)
198 195 dateFile = datetime.date(year, month, day)
199 196
200 197 if (startDate > dateFile) or (endDate < dateFile):
201 198 continue
202 199
203 200 yield thisFile
204 # self.dateFileList.append(dateFile)
205 201
206 202 return
207 203
208 204 def setNextFile(self):
209 205
210 206 if self.online:
211 207 filename = self.search_last_file()
212 208 if not filename:
213 209 self.flagNoMoreFiles = 1
214 210 return 0
215 211 else:
216 212 try:
217 213 filename = next(self.fileList)
218 214 except StopIteration:
219 print('Noooo files')
220 215 self.flagNoMoreFiles = 1
221 216 return 0
222 217
223 218 log.success('Opening {}'.format(filename), 'BLTRParamReader')
224 219
225 220 dirname, name = os.path.split(filename)
226 221 # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
227 222 self.siteFile = filename.split('.')[0]
228 223 if self.filename is not None:
229 224 self.fp.close()
230 225 self.filename = os.path.join(self.path, filename)
231 226 self.fp = open(self.filename, 'rb')
232 227 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
233 228 self.nrecords = self.header_file['nrec'][0]
234 229 self.sizeOfFile = os.path.getsize(self.filename)
235 230 self.counter_records = 0
236 231 self.flagIsNewFile = 0
237 232 self.fileIndex += 1
238 233
239 234 return 1
240 235
241 236 def readNextBlock(self):
242 237
243 238 while True:
244 if self.counter_records == self.nrecords:
239 if not self.online and self.counter_records == self.nrecords:
245 240 self.flagIsNewFile = 1
246 241 if not self.setNextFile():
247 242 return 0
248 243
249 244 try:
250 245 pointer = self.fp.tell()
251 246 self.readBlock()
252 247 except:
253 248 if self.waitDataBlock(pointer, 38512) == 1:
254 249 continue
255 250 else:
256 251 if not self.setNextFile():
257 252 return 0
258 253
259 254 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
260 255 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
261 256 log.warning(
262 257 'Reading Record No. {}/{} -> {} [Skipping]'.format(
263 258 self.counter_records,
264 259 self.nrecords,
265 260 self.datatime.ctime()),
266 261 'BLTRParamReader')
267 262 continue
268 263 break
269 264
270 log.log('Reading Record No. {}/{} -> {}'.format(
265 log.log('Reading Record No. {} -> {}'.format(
271 266 self.counter_records,
272 self.nrecords,
267 # self.nrecords,
273 268 self.datatime.ctime()), 'BLTRParamReader')
274 269
275 270 return 1
276 271
277 272 def readBlock(self):
278 273
279 274 pointer = self.fp.tell()
280 275 header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
281 276 self.nchannels = int(header_rec['nchan'][0] / 2)
282 277 self.kchan = header_rec['nrxs'][0]
283 278 self.nmodes = header_rec['nmodes'][0]
284 279 self.nranges = header_rec['nranges'][0]
285 280 self.fp.seek(pointer)
286 281 self.height = numpy.empty((self.nmodes, self.nranges))
287 282 self.snr = numpy.empty((self.nmodes, int(self.nchannels), self.nranges))
288 283 self.buffer = numpy.empty((self.nmodes, 3, self.nranges))
289 284 self.flagDiscontinuousBlock = 0
290 285
291 286 for mode in range(self.nmodes):
292 287 self.readHeader()
293 288 data = self.readData()
294 289 self.height[mode] = (data[0] - self.correction) / 1000.
295 290 self.buffer[mode] = data[1]
296 291 self.snr[mode] = data[2]
297 292
298 293 self.counter_records = self.counter_records + self.nmodes
299 294
300 295 return
301 296
302 297 def readHeader(self):
303 298 '''
304 299 RecordHeader of BLTR rawdata file
305 300 '''
306 301
307 302 header_structure = numpy.dtype(
308 303 REC_HEADER_STRUCTURE.descr + [
309 304 ('antenna_coord', 'f4', (2, int(self.nchannels))),
310 305 ('rx_gains', 'u4', (int(self.nchannels),)),
311 306 ('rx_analysis', 'u4', (int(self.nchannels),))
312 307 ]
313 308 )
314 309
315 310 self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
316 311 self.lat = self.header_rec['lat'][0]
317 312 self.lon = self.header_rec['lon'][0]
318 313 self.delta = self.header_rec['delta_r'][0]
319 314 self.correction = self.header_rec['dmode_rngcorr'][0]
320 315 self.imode = self.header_rec['dmode_index'][0]
321 316 self.antenna = self.header_rec['antenna_coord']
322 317 self.rx_gains = self.header_rec['rx_gains']
323 318 self.time = self.header_rec['time'][0]
324 319 dt = datetime.datetime.utcfromtimestamp(self.time)
325 320 if dt.date()>self.datatime.date():
326 321 self.flagDiscontinuousBlock = 1
327 322 self.datatime = dt
328 323
329 324 def readData(self):
330 325 '''
331 Reading and filtering data block record of BLTR rawdata file, filtering is according to status_value.
326 Reading and filtering data block record of BLTR rawdata file,
327 filtering is according to status_value.
332 328
333 329 Input:
334 status_value - Array data is set to NAN for values that are not equal to status_value
330 status_value - Array data is set to NAN for values that are not
331 equal to status_value
335 332
336 333 '''
337 334 self.nchannels = int(self.nchannels)
338 335
339 336 data_structure = numpy.dtype(
340 337 DATA_STRUCTURE.descr + [
341 338 ('rx_saturation', 'u4', (self.nchannels,)),
342 339 ('chan_offset', 'u4', (2 * self.nchannels,)),
343 340 ('rx_amp', 'u4', (self.nchannels,)),
344 341 ('rx_snr', 'f4', (self.nchannels,)),
345 342 ('cross_snr', 'f4', (self.kchan,)),
346 343 ('sea_power_relative', 'f4', (self.kchan,))]
347 344 )
348 345
349 346 data = numpy.fromfile(self.fp, data_structure, self.nranges)
350 347
351 348 height = data['range']
352 349 winds = numpy.array(
353 350 (data['zonal'], data['meridional'], data['vertical']))
354 351 snr = data['rx_snr'].T
355 352
356 353 winds[numpy.where(winds == -9999.)] = numpy.nan
357 354 winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
358 355 snr[numpy.where(snr == -9999.)] = numpy.nan
359 356 snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
360 357 snr = numpy.power(10, snr / 10)
361 358
362 359 return height, winds, snr
363 360
364 361 def set_output(self):
365 362 '''
366 363 Storing data from databuffer to dataOut object
367 364 '''
368 365
369 366 self.dataOut.data_SNR = self.snr
370 367 self.dataOut.height = self.height
371 368 self.dataOut.data = self.buffer
372 369 self.dataOut.utctimeInit = self.time
373 370 self.dataOut.utctime = self.dataOut.utctimeInit
374 371 self.dataOut.useLocalTime = False
375 372 self.dataOut.paramInterval = 157
376 373 self.dataOut.timezone = self.timezone
377 374 self.dataOut.site = self.siteFile
378 375 self.dataOut.nrecords = self.nrecords / self.nmodes
379 376 self.dataOut.sizeOfFile = self.sizeOfFile
380 377 self.dataOut.lat = self.lat
381 378 self.dataOut.lon = self.lon
382 379 self.dataOut.channelList = list(range(self.nchannels))
383 380 self.dataOut.kchan = self.kchan
384 381 self.dataOut.delta = self.delta
385 382 self.dataOut.correction = self.correction
386 383 self.dataOut.nmodes = self.nmodes
387 384 self.dataOut.imode = self.imode
388 385 self.dataOut.antenna = self.antenna
389 386 self.dataOut.rx_gains = self.rx_gains
390 387 self.dataOut.flagNoData = False
391 388 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
392 389
393 390 def getData(self):
394 391 '''
395 392 Storing data from databuffer to dataOut object
396 393 '''
397 394 if self.flagNoMoreFiles:
398 395 self.dataOut.flagNoData = True
399 396 self.dataOut.error = 'No More files to read'
400 397 return
401 398
402 399 if not self.readNextBlock():
403 400 self.dataOut.flagNoData = True
404 401 self.dataOut.error = 'Time for wait new file reach!!!'
405 402
406 403 self.set_output()
407 404
408 405 return 1
409 406 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now