##// END OF EJS Templates
schain-jc
RSS
Julia Reader Done!! Task #1085
jespinoza -
r1105:3ff9df258487
parent child
Show More
Side by Side Unified
Context file:
r1105:3ff9df258487 Loading diff...:
Show file before | Show file after | Hide comments
@@ -57,7 +57,7 class PlotData(Operation, Process):
57 57
58 58 Operation.__init__(self, plot=True, **kwargs)
59 59 Process.__init__(self)
60 self.contador = 0
60
61 61 self.kwargs['code'] = self.CODE
62 62 self.mp = False
63 63 self.data = None
@@ -94,7 +94,7 class PlotData(Operation, Process):
94 94 self.height = kwargs.get('height', None)
95 95 self.colorbar = kwargs.get('colorbar', True)
96 96 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
97 self.titles = ['' for __ in range(16)]
97 self.titles = kwargs.get('titles', [])
98 98 self.polar = False
99 99
100 100 def __fmtTime(self, x, pos):
@@ -429,15 +429,15 class PlotData(Operation, Process):
429 429 if self.nrows == 0 or self.nplots == 0:
430 430 log.warning('No data', self.name)
431 431 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
432 fig.canvas.manager.set_window_title(self.CODE)
432 433 continue
433 434
434 435 fig.tight_layout()
435 436 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
436 437 self.getDateTime(self.max_time).strftime('%Y/%m/%d')))
437 # fig.canvas.draw()
438 fig.canvas.draw()
438 439
439 if self.save: # and self.data.ended:
440 self.contador += 1
440 if self.save and self.data.ended:
441 441 channels = range(self.nrows)
442 442 if self.oneFigure:
443 443 label = ''
@@ -445,12 +445,11 class PlotData(Operation, Process):
445 445 label = '_{}'.format(channels[n])
446 446 figname = os.path.join(
447 447 self.save,
448 '{}{}_{}{}.png'.format(
448 '{}{}_{}.png'.format(
449 449 self.CODE,
450 450 label,
451 451 self.getDateTime(self.saveTime).strftime(
452 '%y%m%d_%H%M%S'),
453 str(self.contador),
452 '%y%m%d_%H%M%S'),
454 453 )
455 454 )
456 455 log.log('Saving figure: {}'.format(figname), self.name)
@@ -898,10 +897,11 class PlotParamData(PlotRTIData):
898 897 self.nplots += 1
899 898
900 899 self.ylabel = 'Height [Km]'
901 self.titles = self.data.parameters \
902 if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
903 if self.showSNR:
904 self.titles.append('SNR')
900 if not self.titles:
901 self.titles = self.data.parameters \
902 if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
903 if self.showSNR:
904 self.titles.append('SNR')
905 905
906 906 def plot(self):
907 907 self.data.normalize_heights()
Show file before | Show file after | Hide comments
@@ -32,7 +32,7 FILE_HEADER_STRUCTURE = numpy.dtype([
32 32 REC_HEADER_STRUCTURE = numpy.dtype([
33 33 ('magic', 'f'),
34 34 ('hours', 'f'),
35 ('timedelta', 'f'),
35 ('interval', 'f'),
36 36 ('h0', 'f'),
37 37 ('nheights', 'f'),
38 38 ('snr1', 'f'),
@@ -95,7 +95,7 class JULIAParamReader(JRODataReader, ProcessingUnit):
95 95 self.startTime = startTime
96 96 self.endTime = endTime
97 97 self.datatime = datetime.datetime(1900, 1, 1)
98 self.format = format
98 self.format = format
99 99
100 100 if self.path is None:
101 101 raise ValueError, "The path is not valid"
@@ -174,22 +174,13 class JULIAParamReader(JRODataReader, ProcessingUnit):
174 174 self.fp.close()
175 175 self.filename = filename
176 176 self.fp = open(self.filename, 'rb')
177 self.t0 = [7, 0, 0]
178 self.tf = [18, 0, 0]
179 177
180 178 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
181 179 yy = self.header_file['year'] - 1900 * (self.header_file['year'] > 3000)
182 180 self.year = int(yy + 1900 * (yy < 1000))
183 181 self.doy = int(self.header_file['doy'])
184 self.H0 = self.clockpulse*numpy.round(self.header_file['fh']/self.clockpulse)
185 self.dH = self.clockpulse*numpy.round(self.header_file['dh']/self.clockpulse)
186 self.ipp = self.clockpulse*numpy.round(self.header_file['ipp']/self.clockpulse)
187
188 tau = self.ipp / 1.5E5
189
190 self.nheights = int(self.header_file['nheights'])
191 self.heights = numpy.arange(self.nheights) * self.dH + self.H0
192
182 self.dH = round(self.header_file['dh'], 2)
183 self.ipp = round(self.header_file['ipp'], 2)
193 184 self.sizeOfFile = os.path.getsize(self.filename)
194 185 self.counter_records = 0
195 186 self.flagIsNewFile = 0
@@ -200,12 +191,10 class JULIAParamReader(JRODataReader, ProcessingUnit):
200 191 def readNextBlock(self):
201 192
202 193 while True:
203 if self.fp.tell() == self.sizeOfFile:
194 if not self.readBlock():
204 195 self.flagIsNewFile = 1
205 196 if not self.setNextFile():
206 return 0
207
208 self.readBlock()
197 return 0
209 198
210 199 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
211 200 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
@@ -216,7 +205,7 class JULIAParamReader(JRODataReader, ProcessingUnit):
216 205 self.name)
217 206 continue
218 207 break
219
208
220 209 log.log('Reading Record No. {} -> {}'.format(
221 210 self.counter_records,
222 211 self.datatime.ctime()), self.name)
@@ -225,144 +214,113 class JULIAParamReader(JRODataReader, ProcessingUnit):
225 214
226 215 def readBlock(self):
227 216
228 header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
229 self.timedelta = header_rec['timedelta']
230 if header_rec['magic'] == 888.:
231 h0 = self.clockpulse * numpy.round(header_rec['h0'] / self.clockpulse)
232 nheights = int(header_rec['nheights'])
233 hours = float(header_rec['hours'][0])
234 self.datatime = datetime.datetime(self.year, 1, 1) + datetime.timedelta(days=self.doy-1, hours=hours)
235 self.time = (self.datatime - datetime.datetime(1970,1,1)).total_seconds()
236
237 buffer = numpy.fromfile(self.fp, 'f', 8*nheights).reshape(nheights, 8)
238 idh0 = int(numpy.round((self.H0 - h0) / self.dH))
239 if idh0 == 0 and buffer[0,0] > 1E8:
240 buffer[0,0] = buffer[1,0]
241
242 pow0 = buffer[:, 0]
243 pow1 = buffer[:, 1]
244 acf0 = (buffer[:,2] + buffer[:,3]*1j) / pow0
245 acf1 = (buffer[:,4] + buffer[:,5]*1j) / pow1
246 dccf = (buffer[:,6] + buffer[:,7]*1j) / (pow0*pow1)
247
248 ### SNR
249 sno = (pow0 + pow1 - header_rec['snr']) / header_rec['snr']
250 sno10 = numpy.log10(sno)
251 dsno = 1.0 / numpy.sqrt(self.header_file['nint'] * self.header_file['navg']) * (1 + (1 / sno))
252
253 ### Vertical Drift
254 sp = numpy.sqrt(numpy.abs(acf0)*numpy.abs(acf1))
255 sp[numpy.where(numpy.abs(sp) >= 1.0)] = numpy.sqrt(0.9999)
256
257 vzo = -numpy.arctan2(acf0.imag + acf1.imag,acf0.real + acf1.real)*1.5E5*1.5/(self.ipp*numpy.pi)
258 dvzo = numpy.sqrt(1.0 - sp*sp)*0.338*1.5E5/(numpy.sqrt(self.header_file['nint']*self.header_file['navg'])*sp*self.ipp)
259 err = numpy.where(dvzo <= 0.1)
260 dvzo[err] = 0.1
261
262 #Zonal Drifts
263 dt = self.header_file['nint']*self.ipp / 1.5E5
264 coh = numpy.sqrt(numpy.abs(dccf))
265 err = numpy.where(coh >= 1.0)
266 coh[err] = numpy.sqrt(0.99999)
267
268 err = numpy.where(coh <= 0.1)
269 coh[err] = numpy.sqrt(0.1)
270
271 vxo = numpy.arctan2(dccf.imag, dccf.real)*self.heights[idh0]*1.0E3/(self.kd*dt)
272 dvxo = numpy.sqrt(1.0 - coh*coh)*self.heights[idh0]*1.0E3/(numpy.sqrt(self.header_file['nint']*self.header_file['navg'])*coh*self.kd*dt)
273
274 err = numpy.where(dvxo <= 0.1)
275 dvxo[err] = 0.1
276
277 N = range(len(pow0))
278
279 self.buffer = numpy.zeros((6, self.nheights)) + numpy.nan
280
281 self.buffer[0, idh0+numpy.array(N)] = sno10
282 self.buffer[1, idh0+numpy.array(N)] = vzo
283 self.buffer[2, idh0+numpy.array(N)] = vxo
284 self.buffer[3, idh0+numpy.array(N)] = dvzo
285 self.buffer[4, idh0+numpy.array(N)] = dvxo
286 self.buffer[5, idh0+numpy.array(N)] = dsno
287
288 self.counter_records += 1
217 pointer = self.fp.tell()
218 heights, dt = self.readHeader()
219 self.fp.seek(pointer)
220 buffer_h = []
221 buffer_d = []
222 while True:
223 pointer = self.fp.tell()
224 if pointer == self.sizeOfFile:
225 return 0
226 heights, datatime = self.readHeader()
227 if dt == datatime:
228 buffer_h.append(heights)
229 buffer_d.append(self.readData(len(heights)))
230 continue
231 self.fp.seek(pointer)
232 break
233
234 if dt.date() > self.datatime.date():
235 self.flagDiscontinuousBlock = 1
236 self.datatime = dt
237 self.time = (dt - datetime.datetime(1970, 1, 1)).total_seconds() + time.timezone
238 self.heights = numpy.concatenate(buffer_h)
239 self.buffer = numpy.zeros((5, len(self.heights))) + numpy.nan
240 self.buffer[0, :] = numpy.concatenate([buf[0] for buf in buffer_d])
241 self.buffer[1, :] = numpy.concatenate([buf[1] for buf in buffer_d])
242 self.buffer[2, :] = numpy.concatenate([buf[2] for buf in buffer_d])
243 self.buffer[3, :] = numpy.concatenate([buf[3] for buf in buffer_d])
244 self.buffer[4, :] = numpy.concatenate([buf[4] for buf in buffer_d])
245
246 self.counter_records += 1
289 247
290 return
248 return 1
291 249
292 250 def readHeader(self):
293 251 '''
294 RecordHeader of BLTR rawdata file
252 Parse recordHeader
295 253 '''
296
297 header_structure = numpy.dtype(
298 REC_HEADER_STRUCTURE.descr + [
299 ('antenna_coord', 'f4', (2, self.nchannels)),
300 ('rx_gains', 'u4', (self.nchannels,)),
301 ('rx_analysis', 'u4', (self.nchannels,))
302 ]
303 )
304
305 self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
306 self.lat = self.header_rec['lat'][0]
307 self.lon = self.header_rec['lon'][0]
308 self.delta = self.header_rec['delta_r'][0]
309 self.correction = self.header_rec['dmode_rngcorr'][0]
310 self.imode = self.header_rec['dmode_index'][0]
311 self.antenna = self.header_rec['antenna_coord']
312 self.rx_gains = self.header_rec['rx_gains']
313 self.time = self.header_rec['time'][0]
314 dt = datetime.datetime.utcfromtimestamp(self.time)
315 if dt.date()>self.datatime.date():
316 self.flagDiscontinuousBlock = 1
317 self.datatime = dt
318 254
319 def readData(self):
255 self.header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
256 self.interval = self.header_rec['interval']
257 if self.header_rec['magic'] == 888.:
258 self.header_rec['h0'] = round(self.header_rec['h0'], 2)
259 nheights = int(self.header_rec['nheights'])
260 hours = float(self.header_rec['hours'][0])
261 heights = numpy.arange(nheights) * self.dH + self.header_rec['h0']
262 datatime = datetime.datetime(self.year, 1, 1) + datetime.timedelta(days=self.doy-1, hours=hours)
263 return heights, datatime
264 else:
265 return False
266
267 def readData(self, N):
320 268 '''
321 Reading and filtering data block record of BLTR rawdata file, filtering is according to status_value.
322
323 Input:
324 status_value - Array data is set to NAN for values that are not equal to status_value
325
269 Parse data
326 270 '''
327 271
328 data_structure = numpy.dtype(
329 DATA_STRUCTURE.descr + [
330 ('rx_saturation', 'u4', (self.nchannels,)),
331 ('chan_offset', 'u4', (2 * self.nchannels,)),
332 ('rx_amp', 'u4', (self.nchannels,)),
333 ('rx_snr', 'f4', (self.nchannels,)),
334 ('cross_snr', 'f4', (self.kchan,)),
335 ('sea_power_relative', 'f4', (self.kchan,))]
336 )
337
338 data = numpy.fromfile(self.fp, data_structure, self.nranges)
339
340 height = data['range']
341 winds = numpy.array(
342 (data['zonal'], data['meridional'], data['vertical']))
343 snr = data['rx_snr'].T
344
345 winds[numpy.where(winds == -9999.)] = numpy.nan
346 winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
347 snr[numpy.where(snr == -9999.)] = numpy.nan
348 snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
349 snr = numpy.power(10, snr / 10)
272 buffer = numpy.fromfile(self.fp, 'f', 8*N).reshape(N, 8)
273
274 pow0 = buffer[:, 0]
275 pow1 = buffer[:, 1]
276 acf0 = (buffer[:,2] + buffer[:,3]*1j) / pow0
277 acf1 = (buffer[:,4] + buffer[:,5]*1j) / pow1
278 dccf = (buffer[:,6] + buffer[:,7]*1j) / (pow0*pow1)
279
280 ### SNR
281 sno = (pow0 + pow1 - self.header_rec['snr']) / self.header_rec['snr']
282 sno10 = numpy.log10(sno)
283 # dsno = 1.0 / numpy.sqrt(self.header_file['nint'] * self.header_file['navg']) * (1 + (1 / sno))
284
285 ### Vertical Drift
286 sp = numpy.sqrt(numpy.abs(acf0)*numpy.abs(acf1))
287 sp[numpy.where(numpy.abs(sp) >= 1.0)] = numpy.sqrt(0.9999)
288
289 vzo = -numpy.arctan2(acf0.imag + acf1.imag,acf0.real + acf1.real)*1.5E5*1.5/(self.ipp*numpy.pi)
290 dvzo = numpy.sqrt(1.0 - sp*sp)*0.338*1.5E5/(numpy.sqrt(self.header_file['nint']*self.header_file['navg'])*sp*self.ipp)
291 err = numpy.where(dvzo <= 0.1)
292 dvzo[err] = 0.1
293
294 #Zonal Drifts
295 dt = self.header_file['nint']*self.ipp / 1.5E5
296 coh = numpy.sqrt(numpy.abs(dccf))
297 err = numpy.where(coh >= 1.0)
298 coh[err] = numpy.sqrt(0.99999)
299
300 err = numpy.where(coh <= 0.1)
301 coh[err] = numpy.sqrt(0.1)
302
303 vxo = numpy.arctan2(dccf.imag, dccf.real)*self.header_rec['h0']*1.0E3/(self.kd*dt)
304 dvxo = numpy.sqrt(1.0 - coh*coh)*self.header_rec['h0']*1.0E3/(numpy.sqrt(self.header_file['nint']*self.header_file['navg'])*coh*self.kd*dt)
305
306 err = numpy.where(dvxo <= 0.1)
307 dvxo[err] = 0.1
350 308
351 return height, winds, snr
309 return vzo, dvzo, vxo, dvxo, sno10
352 310
353 311 def set_output(self):
354 312 '''
355 313 Storing data from databuffer to dataOut object
356 314 '''
357
358 self.dataOut.data_SNR = self.buffer[0]
359 self.dataOut.heights = self.heights
360 self.dataOut.data_param = self.buffer[1:,]
315
316 self.dataOut.data_SNR = self.buffer[4].reshape(1, -1)
317 self.dataOut.heightList = self.heights
318 self.dataOut.data_param = self.buffer[0:4,]
361 319 self.dataOut.utctimeInit = self.time
362 self.dataOut.utctime = self.dataOut.utctimeInit
363 self.dataOut.useLocalTime = False
364 self.dataOut.paramInterval = self.timedelta
365 self.dataOut.timezone = self.timezone
320 self.dataOut.utctime = self.time
321 self.dataOut.useLocalTime = True
322 self.dataOut.paramInterval = self.interval
323 self.dataOut.timezone = self.timezone
366 324 self.dataOut.sizeOfFile = self.sizeOfFile
367 325 self.dataOut.flagNoData = False
368 326 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
Show file before | Show file after | Hide comments
@@ -138,12 +138,11 class Data(object):
138 138 return self.data[key][self.__times[0]].shape
139 139 return (0,)
140 140
141 def update(self, dataOut):
141 def update(self, dataOut, tm):
142 142 '''
143 143 Update data object with new dataOut
144 144 '''
145
146 tm = dataOut.utctime
145
147 146 if tm in self.__times:
148 147 return
149 148
@@ -578,7 +577,7 class PlotterReceiver(ProcessingUnit, Process):
578 577 while True:
579 578 dataOut = self.receiver.recv_pyobj()
580 579 if not dataOut.flagNoData:
581 if dataOut.type == 'Parameters':
580 if dataOut.type == 'Parameters':
582 581 tm = dataOut.utctimeInit
583 582 else:
584 583 tm = dataOut.utctime
@@ -599,7 +598,7 class PlotterReceiver(ProcessingUnit, Process):
599 598 self.data.setup()
600 599 self.dates.append(dt)
601 600
602 self.data.update(dataOut)
601 self.data.update(dataOut, tm)
603 602
604 603 if dataOut.finished is True:
605 604 self.connections -= 1
General Comments 0
You need to be logged in to leave comments. Login now