##// END OF EJS Templates
schain-jc
RSS
Julia Reader Done!! Task #1085
jespinoza -
r1105:3ff9df258487
parent child
Show More
Side by Side Unified
Context file:
r1105:3ff9df258487 Loading diff...:
Show file before | Show file after | Hide comments
@@ -57,7 +57,7 class PlotData(Operation, Process):
57
57
58 Operation.__init__(self, plot=True, **kwargs)
58 Operation.__init__(self, plot=True, **kwargs)
59 Process.__init__(self)
59 Process.__init__(self)
60 self.contador = 0
60
61 self.kwargs['code'] = self.CODE
61 self.kwargs['code'] = self.CODE
62 self.mp = False
62 self.mp = False
63 self.data = None
63 self.data = None
@@ -94,7 +94,7 class PlotData(Operation, Process):
94 self.height = kwargs.get('height', None)
94 self.height = kwargs.get('height', None)
95 self.colorbar = kwargs.get('colorbar', True)
95 self.colorbar = kwargs.get('colorbar', True)
96 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
96 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
97 self.titles = ['' for __ in range(16)]
97 self.titles = kwargs.get('titles', [])
98 self.polar = False
98 self.polar = False
99
99
100 def __fmtTime(self, x, pos):
100 def __fmtTime(self, x, pos):
@@ -429,15 +429,15 class PlotData(Operation, Process):
429 if self.nrows == 0 or self.nplots == 0:
429 if self.nrows == 0 or self.nplots == 0:
430 log.warning('No data', self.name)
430 log.warning('No data', self.name)
431 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
431 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
432 fig.canvas.manager.set_window_title(self.CODE)
432 continue
433 continue
433
434
434 fig.tight_layout()
435 fig.tight_layout()
435 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
436 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
436 self.getDateTime(self.max_time).strftime('%Y/%m/%d')))
437 self.getDateTime(self.max_time).strftime('%Y/%m/%d')))
437 # fig.canvas.draw()
438 fig.canvas.draw()
438
439
439 if self.save: # and self.data.ended:
440 if self.save and self.data.ended:
440 self.contador += 1
441 channels = range(self.nrows)
441 channels = range(self.nrows)
442 if self.oneFigure:
442 if self.oneFigure:
443 label = ''
443 label = ''
@@ -445,12 +445,11 class PlotData(Operation, Process):
445 label = '_{}'.format(channels[n])
445 label = '_{}'.format(channels[n])
446 figname = os.path.join(
446 figname = os.path.join(
447 self.save,
447 self.save,
448 '{}{}_{}{}.png'.format(
448 '{}{}_{}.png'.format(
449 self.CODE,
449 self.CODE,
450 label,
450 label,
451 self.getDateTime(self.saveTime).strftime(
451 self.getDateTime(self.saveTime).strftime(
452 '%y%m%d_%H%M%S'),
452 '%y%m%d_%H%M%S'),
453 str(self.contador),
454 )
453 )
455 )
454 )
456 log.log('Saving figure: {}'.format(figname), self.name)
455 log.log('Saving figure: {}'.format(figname), self.name)
@@ -898,6 +897,7 class PlotParamData(PlotRTIData):
898 self.nplots += 1
897 self.nplots += 1
899
898
900 self.ylabel = 'Height [Km]'
899 self.ylabel = 'Height [Km]'
900 if not self.titles:
901 self.titles = self.data.parameters \
901 self.titles = self.data.parameters \
902 if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
902 if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
903 if self.showSNR:
903 if self.showSNR:
Show file before | Show file after | Hide comments
@@ -32,7 +32,7 FILE_HEADER_STRUCTURE = numpy.dtype([
32 REC_HEADER_STRUCTURE = numpy.dtype([
32 REC_HEADER_STRUCTURE = numpy.dtype([
33 ('magic', 'f'),
33 ('magic', 'f'),
34 ('hours', 'f'),
34 ('hours', 'f'),
35 ('timedelta', 'f'),
35 ('interval', 'f'),
36 ('h0', 'f'),
36 ('h0', 'f'),
37 ('nheights', 'f'),
37 ('nheights', 'f'),
38 ('snr1', 'f'),
38 ('snr1', 'f'),
@@ -174,22 +174,13 class JULIAParamReader(JRODataReader, ProcessingUnit):
174 self.fp.close()
174 self.fp.close()
175 self.filename = filename
175 self.filename = filename
176 self.fp = open(self.filename, 'rb')
176 self.fp = open(self.filename, 'rb')
177 self.t0 = [7, 0, 0]
178 self.tf = [18, 0, 0]
179
177
180 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
178 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
181 yy = self.header_file['year'] - 1900 * (self.header_file['year'] > 3000)
179 yy = self.header_file['year'] - 1900 * (self.header_file['year'] > 3000)
182 self.year = int(yy + 1900 * (yy < 1000))
180 self.year = int(yy + 1900 * (yy < 1000))
183 self.doy = int(self.header_file['doy'])
181 self.doy = int(self.header_file['doy'])
184 self.H0 = self.clockpulse*numpy.round(self.header_file['fh']/self.clockpulse)
182 self.dH = round(self.header_file['dh'], 2)
185 self.dH = self.clockpulse*numpy.round(self.header_file['dh']/self.clockpulse)
183 self.ipp = round(self.header_file['ipp'], 2)
186 self.ipp = self.clockpulse*numpy.round(self.header_file['ipp']/self.clockpulse)
187
188 tau = self.ipp / 1.5E5
189
190 self.nheights = int(self.header_file['nheights'])
191 self.heights = numpy.arange(self.nheights) * self.dH + self.H0
192
193 self.sizeOfFile = os.path.getsize(self.filename)
184 self.sizeOfFile = os.path.getsize(self.filename)
194 self.counter_records = 0
185 self.counter_records = 0
195 self.flagIsNewFile = 0
186 self.flagIsNewFile = 0
@@ -200,13 +191,11 class JULIAParamReader(JRODataReader, ProcessingUnit):
200 def readNextBlock(self):
191 def readNextBlock(self):
201
192
202 while True:
193 while True:
203 if self.fp.tell() == self.sizeOfFile:
194 if not self.readBlock():
204 self.flagIsNewFile = 1
195 self.flagIsNewFile = 1
205 if not self.setNextFile():
196 if not self.setNextFile():
206 return 0
197 return 0
207
198
208 self.readBlock()
209
210 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
199 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
211 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
200 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
212 log.warning(
201 log.warning(
@@ -225,19 +214,62 class JULIAParamReader(JRODataReader, ProcessingUnit):
225
214
226 def readBlock(self):
215 def readBlock(self):
227
216
228 header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
217 pointer = self.fp.tell()
229 self.timedelta = header_rec['timedelta']
218 heights, dt = self.readHeader()
230 if header_rec['magic'] == 888.:
219 self.fp.seek(pointer)
231 h0 = self.clockpulse * numpy.round(header_rec['h0'] / self.clockpulse)
220 buffer_h = []
232 nheights = int(header_rec['nheights'])
221 buffer_d = []
233 hours = float(header_rec['hours'][0])
222 while True:
234 self.datatime = datetime.datetime(self.year, 1, 1) + datetime.timedelta(days=self.doy-1, hours=hours)
223 pointer = self.fp.tell()
235 self.time = (self.datatime - datetime.datetime(1970,1,1)).total_seconds()
224 if pointer == self.sizeOfFile:
225 return 0
226 heights, datatime = self.readHeader()
227 if dt == datatime:
228 buffer_h.append(heights)
229 buffer_d.append(self.readData(len(heights)))
230 continue
231 self.fp.seek(pointer)
232 break
233
234 if dt.date() > self.datatime.date():
235 self.flagDiscontinuousBlock = 1
236 self.datatime = dt
237 self.time = (dt - datetime.datetime(1970, 1, 1)).total_seconds() + time.timezone
238 self.heights = numpy.concatenate(buffer_h)
239 self.buffer = numpy.zeros((5, len(self.heights))) + numpy.nan
240 self.buffer[0, :] = numpy.concatenate([buf[0] for buf in buffer_d])
241 self.buffer[1, :] = numpy.concatenate([buf[1] for buf in buffer_d])
242 self.buffer[2, :] = numpy.concatenate([buf[2] for buf in buffer_d])
243 self.buffer[3, :] = numpy.concatenate([buf[3] for buf in buffer_d])
244 self.buffer[4, :] = numpy.concatenate([buf[4] for buf in buffer_d])
245
246 self.counter_records += 1
247
248 return 1
249
250 def readHeader(self):
251 '''
252 Parse recordHeader
253 '''
254
255 self.header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
256 self.interval = self.header_rec['interval']
257 if self.header_rec['magic'] == 888.:
258 self.header_rec['h0'] = round(self.header_rec['h0'], 2)
259 nheights = int(self.header_rec['nheights'])
260 hours = float(self.header_rec['hours'][0])
261 heights = numpy.arange(nheights) * self.dH + self.header_rec['h0']
262 datatime = datetime.datetime(self.year, 1, 1) + datetime.timedelta(days=self.doy-1, hours=hours)
263 return heights, datatime
264 else:
265 return False
266
267 def readData(self, N):
268 '''
269 Parse data
270 '''
236
271
237 buffer = numpy.fromfile(self.fp, 'f', 8*nheights).reshape(nheights, 8)
272 buffer = numpy.fromfile(self.fp, 'f', 8*N).reshape(N, 8)
238 idh0 = int(numpy.round((self.H0 - h0) / self.dH))
239 if idh0 == 0 and buffer[0,0] > 1E8:
240 buffer[0,0] = buffer[1,0]
241
273
242 pow0 = buffer[:, 0]
274 pow0 = buffer[:, 0]
243 pow1 = buffer[:, 1]
275 pow1 = buffer[:, 1]
@@ -246,9 +278,9 class JULIAParamReader(JRODataReader, ProcessingUnit):
246 dccf = (buffer[:,6] + buffer[:,7]*1j) / (pow0*pow1)
278 dccf = (buffer[:,6] + buffer[:,7]*1j) / (pow0*pow1)
247
279
248 ### SNR
280 ### SNR
249 sno = (pow0 + pow1 - header_rec['snr']) / header_rec['snr']
281 sno = (pow0 + pow1 - self.header_rec['snr']) / self.header_rec['snr']
250 sno10 = numpy.log10(sno)
282 sno10 = numpy.log10(sno)
251 dsno = 1.0 / numpy.sqrt(self.header_file['nint'] * self.header_file['navg']) * (1 + (1 / sno))
283 # dsno = 1.0 / numpy.sqrt(self.header_file['nint'] * self.header_file['navg']) * (1 + (1 / sno))
252
284
253 ### Vertical Drift
285 ### Vertical Drift
254 sp = numpy.sqrt(numpy.abs(acf0)*numpy.abs(acf1))
286 sp = numpy.sqrt(numpy.abs(acf0)*numpy.abs(acf1))
@@ -268,100 +300,26 class JULIAParamReader(JRODataReader, ProcessingUnit):
268 err = numpy.where(coh <= 0.1)
300 err = numpy.where(coh <= 0.1)
269 coh[err] = numpy.sqrt(0.1)
301 coh[err] = numpy.sqrt(0.1)
270
302
271 vxo = numpy.arctan2(dccf.imag, dccf.real)*self.heights[idh0]*1.0E3/(self.kd*dt)
303 vxo = numpy.arctan2(dccf.imag, dccf.real)*self.header_rec['h0']*1.0E3/(self.kd*dt)
272 dvxo = numpy.sqrt(1.0 - coh*coh)*self.heights[idh0]*1.0E3/(numpy.sqrt(self.header_file['nint']*self.header_file['navg'])*coh*self.kd*dt)
304 dvxo = numpy.sqrt(1.0 - coh*coh)*self.header_rec['h0']*1.0E3/(numpy.sqrt(self.header_file['nint']*self.header_file['navg'])*coh*self.kd*dt)
273
305
274 err = numpy.where(dvxo <= 0.1)
306 err = numpy.where(dvxo <= 0.1)
275 dvxo[err] = 0.1
307 dvxo[err] = 0.1
276
308
277 N = range(len(pow0))
309 return vzo, dvzo, vxo, dvxo, sno10
278
279 self.buffer = numpy.zeros((6, self.nheights)) + numpy.nan
280
281 self.buffer[0, idh0+numpy.array(N)] = sno10
282 self.buffer[1, idh0+numpy.array(N)] = vzo
283 self.buffer[2, idh0+numpy.array(N)] = vxo
284 self.buffer[3, idh0+numpy.array(N)] = dvzo
285 self.buffer[4, idh0+numpy.array(N)] = dvxo
286 self.buffer[5, idh0+numpy.array(N)] = dsno
287
288 self.counter_records += 1
289
290 return
291
292 def readHeader(self):
293 '''
294 RecordHeader of BLTR rawdata file
295 '''
296
297 header_structure = numpy.dtype(
298 REC_HEADER_STRUCTURE.descr + [
299 ('antenna_coord', 'f4', (2, self.nchannels)),
300 ('rx_gains', 'u4', (self.nchannels,)),
301 ('rx_analysis', 'u4', (self.nchannels,))
302 ]
303 )
304
305 self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
306 self.lat = self.header_rec['lat'][0]
307 self.lon = self.header_rec['lon'][0]
308 self.delta = self.header_rec['delta_r'][0]
309 self.correction = self.header_rec['dmode_rngcorr'][0]
310 self.imode = self.header_rec['dmode_index'][0]
311 self.antenna = self.header_rec['antenna_coord']
312 self.rx_gains = self.header_rec['rx_gains']
313 self.time = self.header_rec['time'][0]
314 dt = datetime.datetime.utcfromtimestamp(self.time)
315 if dt.date()>self.datatime.date():
316 self.flagDiscontinuousBlock = 1
317 self.datatime = dt
318
319 def readData(self):
320 '''
321 Reading and filtering data block record of BLTR rawdata file, filtering is according to status_value.
322
323 Input:
324 status_value - Array data is set to NAN for values that are not equal to status_value
325
326 '''
327
328 data_structure = numpy.dtype(
329 DATA_STRUCTURE.descr + [
330 ('rx_saturation', 'u4', (self.nchannels,)),
331 ('chan_offset', 'u4', (2 * self.nchannels,)),
332 ('rx_amp', 'u4', (self.nchannels,)),
333 ('rx_snr', 'f4', (self.nchannels,)),
334 ('cross_snr', 'f4', (self.kchan,)),
335 ('sea_power_relative', 'f4', (self.kchan,))]
336 )
337
338 data = numpy.fromfile(self.fp, data_structure, self.nranges)
339
340 height = data['range']
341 winds = numpy.array(
342 (data['zonal'], data['meridional'], data['vertical']))
343 snr = data['rx_snr'].T
344
345 winds[numpy.where(winds == -9999.)] = numpy.nan
346 winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
347 snr[numpy.where(snr == -9999.)] = numpy.nan
348 snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
349 snr = numpy.power(10, snr / 10)
350
351 return height, winds, snr
352
310
353 def set_output(self):
311 def set_output(self):
354 '''
312 '''
355 Storing data from databuffer to dataOut object
313 Storing data from databuffer to dataOut object
356 '''
314 '''
357
315
358 self.dataOut.data_SNR = self.buffer[0]
316 self.dataOut.data_SNR = self.buffer[4].reshape(1, -1)
359 self.dataOut.heights = self.heights
317 self.dataOut.heightList = self.heights
360 self.dataOut.data_param = self.buffer[1:,]
318 self.dataOut.data_param = self.buffer[0:4,]
361 self.dataOut.utctimeInit = self.time
319 self.dataOut.utctimeInit = self.time
362 self.dataOut.utctime = self.dataOut.utctimeInit
320 self.dataOut.utctime = self.time
363 self.dataOut.useLocalTime = False
321 self.dataOut.useLocalTime = True
364 self.dataOut.paramInterval = self.timedelta
322 self.dataOut.paramInterval = self.interval
365 self.dataOut.timezone = self.timezone
323 self.dataOut.timezone = self.timezone
366 self.dataOut.sizeOfFile = self.sizeOfFile
324 self.dataOut.sizeOfFile = self.sizeOfFile
367 self.dataOut.flagNoData = False
325 self.dataOut.flagNoData = False
Show file before | Show file after | Hide comments
@@ -138,12 +138,11 class Data(object):
138 return self.data[key][self.__times[0]].shape
138 return self.data[key][self.__times[0]].shape
139 return (0,)
139 return (0,)
140
140
141 def update(self, dataOut):
141 def update(self, dataOut, tm):
142 '''
142 '''
143 Update data object with new dataOut
143 Update data object with new dataOut
144 '''
144 '''
145
145
146 tm = dataOut.utctime
147 if tm in self.__times:
146 if tm in self.__times:
148 return
147 return
149
148
@@ -599,7 +598,7 class PlotterReceiver(ProcessingUnit, Process):
599 self.data.setup()
598 self.data.setup()
600 self.dates.append(dt)
599 self.dates.append(dt)
601
600
602 self.data.update(dataOut)
601 self.data.update(dataOut, tm)
603
602
604 if dataOut.finished is True:
603 if dataOut.finished is True:
605 self.connections -= 1
604 self.connections -= 1
General Comments 0
You need to be logged in to leave comments. Login now