schain-jc Commit - r1105:3ff9df258487

Julia Reader Done!! Task #1085

jespinoza -

r1105:3ff9df258487

parent child

Context file:

r1105:3ff9df258487

Collapse all files

schainpy/model/graphics/jroplot_data.py +12 -12

                     Operation.__init__(self, plot=True, **kwargs)
                     Process.__init__(self)
-                    self.contador = 0
                     self.kwargs['code'] = self.CODE
                     self.mp = False
                     self.data = None
                     self.height = kwargs.get('height', None)
                     self.colorbar = kwargs.get('colorbar', True)
                     self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
-                    self.titles = ['' for __ in range(16)]
+                    self.titles = kwargs.get('titles', [])
                     self.polar = False
                 def __fmtTime(self, x, pos):
                         if self.nrows == 0 or self.nplots == 0:
                             log.warning('No data', self.name)
                             fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
+                            fig.canvas.manager.set_window_title(self.CODE)
                             continue
                         fig.tight_layout()
                         fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
                                                                              self.getDateTime(self.max_time).strftime('%Y/%m/%d')))
-                        # fig.canvas.draw()
+                        fig.canvas.draw()
-                        if self.save:  # and self.data.ended:
+                        if self.save and self.data.ended:
-                            self.contador += 1
                             channels = range(self.nrows)
                             if self.oneFigure:
                                 label = ''
                                 label = '_{}'.format(channels[n])
                             figname = os.path.join(
                                 self.save,
-                                '{}{}_{}{}.png'.format(
+                                '{}{}_{}.png'.format(
                                     self.CODE,
                                     label,
                                     self.getDateTime(self.saveTime).strftime(
                                         '%y%m%d_%H%M%S'),
-                                    str(self.contador),
                                 )
                             )
                             log.log('Saving figure: {}'.format(figname), self.name)
                         self.nplots += 1
                     self.ylabel = 'Height [Km]'
-                    self.titles = self.data.parameters \
+                    if not self.titles:
-                        if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
+                        self.titles = self.data.parameters \
-                    if self.showSNR:
+                            if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
-                        self.titles.append('SNR')
+                        if self.showSNR:
+                            self.titles.append('SNR')
                 def plot(self):
                     self.data.normalize_heights()

schainpy/model/io/julIO_param.py +98 -140

@@ -32,7 +32,7 FILE_HEADER_STRUCTURE = numpy.dtype([
32	REC_HEADER_STRUCTURE = numpy.dtype([	32	REC_HEADER_STRUCTURE = numpy.dtype([
33	('magic', 'f'),	33	('magic', 'f'),
34	('hours', 'f'),	34	('hours', 'f'),
35	('~~timedelta~~', 'f'),	35	('interval', 'f'),
36	('h0', 'f'),	36	('h0', 'f'),
37	('nheights', 'f'),	37	('nheights', 'f'),
38	('snr1', 'f'),	38	('snr1', 'f'),
@@ -95,7 +95,7 class JULIAParamReader(JRODataReader, ProcessingUnit):
95	self.startTime = startTime	95	self.startTime = startTime
96	self.endTime = endTime	96	self.endTime = endTime
97	self.datatime = datetime.datetime(1900, 1, 1)	97	self.datatime = datetime.datetime(1900, 1, 1)
98	self.format = format	98	self.format = format
99		99
100	if self.path is None:	100	if self.path is None:
101	raise ValueError, "The path is not valid"	101	raise ValueError, "The path is not valid"
@@ -174,22 +174,13 class JULIAParamReader(JRODataReader, ProcessingUnit):
174	self.fp.close()	174	self.fp.close()
175	self.filename = filename	175	self.filename = filename
176	self.fp = open(self.filename, 'rb')	176	self.fp = open(self.filename, 'rb')
177	self.t0 = [7, 0, 0]
178	self.tf = [18, 0, 0]
179		177
180	self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)	178	self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
181	yy = self.header_file['year'] - 1900 * (self.header_file['year'] > 3000)	179	yy = self.header_file['year'] - 1900 * (self.header_file['year'] > 3000)
182	self.year = int(yy + 1900 * (yy < 1000))	180	self.year = int(yy + 1900 * (yy < 1000))
183	self.doy = int(self.header_file['doy'])	181	self.doy = int(self.header_file['doy'])
184	self.H0 = ~~self~~.~~clockpulse~~*~~numpy~~.round(self.header_file['fh']/~~self~~.~~clockpulse~~)	182	self.dH = round(self.header_file['dh'], 2)
185	self.dH = self.clockpulse*numpy.round(self.header_file['dh']/self.clockpulse)	183	self.ipp = round(self.header_file['ipp'], 2)
186	self.ipp = self.clockpulse*numpy.round(self.header_file['ipp']/self.clockpulse)
187
188	tau = self.ipp / 1.5E5
189
190	self.nheights = int(self.header_file['nheights'])
191	self.heights = numpy.arange(self.nheights) * self.dH + self.H0
192
193	self.sizeOfFile = os.path.getsize(self.filename)	184	self.sizeOfFile = os.path.getsize(self.filename)
194	self.counter_records = 0	185	self.counter_records = 0
195	self.flagIsNewFile = 0	186	self.flagIsNewFile = 0
@@ -200,12 +191,10 class JULIAParamReader(JRODataReader, ProcessingUnit):
200	def readNextBlock(self):	191	def readNextBlock(self):
201		192
202	while True:	193	while True:
203	if self.fp.tell() == self.sizeOfFile:	194	if not self.readBlock():
204	self.flagIsNewFile = 1	195	self.flagIsNewFile = 1
205	if not self.setNextFile():	196	if not self.setNextFile():
206	return 0	197	return 0
207
208	self.readBlock()
209		198
210	if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \	199	if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
211	(self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):	200	(self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
@@ -216,7 +205,7 class JULIAParamReader(JRODataReader, ProcessingUnit):
216	self.name)	205	self.name)
217	continue	206	continue
218	break	207	break
219		208
220	log.log('Reading Record No. {} -> {}'.format(	209	log.log('Reading Record No. {} -> {}'.format(
221	self.counter_records,	210	self.counter_records,
222	self.datatime.ctime()), self.name)	211	self.datatime.ctime()), self.name)
@@ -225,144 +214,113 class JULIAParamReader(JRODataReader, ProcessingUnit):
225		214
226	def readBlock(self):	215	def readBlock(self):
227		216
228	header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)	217	pointer = self.fp.tell()
229	self.timedelta = header_rec['timedelta']	218	heights, dt = self.readHeader()
230	if header_rec['magic'] == 888.:	219	self.fp.seek(pointer)
231	h0 = self.clockpulse * numpy.round(header_rec['h0'] / self.clockpulse)	220	buffer_h = []
232	nheights = int(header_rec['nheights'])	221	buffer_d = []
233	hours = float(header_rec['hours'][0])	222	while True:
234	self.datatime = datetime.datetime(self.year, 1, 1) + datetime.timedelta(days=self.doy-1, hours=hours)	223	pointer = self.fp.tell()
235	self.time = (self.datatime - datetime.datetime(1970,1,1)).total_seconds()	224	if pointer == self.sizeOfFile:
236		225	return 0
237	buffer = numpy.fromfile(self.fp, 'f', 8*nheights).reshape(nheights, 8)	226	heights, datatime = self.readHeader()
238	idh0 = int(numpy.round((self.H0 - h0) / self.dH))	227	if dt == datatime:
239	if idh0 == 0 and buffer[0,0] > 1E8:	228	buffer_h.append(heights)
240	buffer[0,0] = buffer[1,0]	229	buffer_d.append(self.readData(len(heights)))
241		230	continue
242	pow0 = buffer[:, 0]	231	self.fp.seek(pointer)
243	pow1 = buffer[:, 1]	232	break
244	acf0 = (buffer[:,2] + buffer[:,3]*1j) / pow0	233
245	acf1 = (buffer[:,4] + buffer[:,5]*1j) / pow1	234	if dt.date() > self.datatime.date():
246	dccf = (buffer[:,6] + buffer[:,7]1j) / (pow0pow1)	235	self.flagDiscontinuousBlock = 1
247		236	self.datatime = dt
248	### SNR	237	self.time = (dt - datetime.datetime(1970, 1, 1)).total_seconds() + time.timezone
249	sno = (pow0 + pow1 - header_rec['snr']) / header_rec['snr']	238	self.heights = numpy.concatenate(buffer_h)
250	sno10 = numpy.log10(sno)	239	self.buffer = numpy.zeros((5, len(self.heights))) + numpy.nan
251	dsno = 1.0 / numpy.sqrt(self.header_file['nint'] * self.header_file['navg']) * (1 + (1 / sno))	240	self.buffer[0, :] = numpy.concatenate([buf[0] for buf in buffer_d])
252		241	self.buffer[1, :] = numpy.concatenate([buf[1] for buf in buffer_d])
253	### Vertical Drift	242	self.buffer[2, :] = numpy.concatenate([buf[2] for buf in buffer_d])
254	sp = numpy.sqrt(numpy.abs(acf0)*numpy.abs(acf1))	243	self.buffer[3, :] = numpy.concatenate([buf[3] for buf in buffer_d])
255	sp[numpy.where(numpy.abs(sp) >= 1.0)] = numpy.sqrt(0.9999)	244	self.buffer[4, :] = numpy.concatenate([buf[4] for buf in buffer_d])
256		245
257	vzo = -numpy.arctan2(acf0.imag + acf1.imag,acf0.real + acf1.real)1.5E51.5/(self.ipp*numpy.pi)	246	self.counter_records += 1
258	dvzo = numpy.sqrt(1.0 - spsp)0.3381.5E5/(numpy.sqrt(self.header_file['nint']self.header_file['navg'])spself.ipp)
259	err = numpy.where(dvzo <= 0.1)
260	dvzo[err] = 0.1
261
262	#Zonal Drifts
263	dt = self.header_file['nint']*self.ipp / 1.5E5
264	coh = numpy.sqrt(numpy.abs(dccf))
265	err = numpy.where(coh >= 1.0)
266	coh[err] = numpy.sqrt(0.99999)
267
268	err = numpy.where(coh <= 0.1)
269	coh[err] = numpy.sqrt(0.1)
270
271	vxo = numpy.arctan2(dccf.imag, dccf.real)self.heights[idh0]1.0E3/(self.kd*dt)
272	dvxo = numpy.sqrt(1.0 - cohcoh)self.heights[idh0]1.0E3/(numpy.sqrt(self.header_file['nint']self.header_file['navg'])cohself.kd*dt)
273
274	err = numpy.where(dvxo <= 0.1)
275	dvxo[err] = 0.1
276
277	N = range(len(pow0))
278
279	self.buffer = numpy.zeros((6, self.nheights)) + numpy.nan
280
281	self.buffer[0, idh0+numpy.array(N)] = sno10
282	self.buffer[1, idh0+numpy.array(N)] = vzo
283	self.buffer[2, idh0+numpy.array(N)] = vxo
284	self.buffer[3, idh0+numpy.array(N)] = dvzo
285	self.buffer[4, idh0+numpy.array(N)] = dvxo
286	self.buffer[5, idh0+numpy.array(N)] = dsno
287
288	self.counter_records += 1
289		247
290	return	248	return 1
291		249
292	def readHeader(self):	250	def readHeader(self):
293	'''	251	'''
294	RecordHeader of BLTR rawdata file	252	Parse recordHeader
295	'''	253	'''
296
297	header_structure = numpy.dtype(
298	REC_HEADER_STRUCTURE.descr + [
299	('antenna_coord', 'f4', (2, self.nchannels)),
300	('rx_gains', 'u4', (self.nchannels,)),
301	('rx_analysis', 'u4', (self.nchannels,))
302	]
303	)
304
305	self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
306	self.lat = self.header_rec['lat'][0]
307	self.lon = self.header_rec['lon'][0]
308	self.delta = self.header_rec['delta_r'][0]
309	self.correction = self.header_rec['dmode_rngcorr'][0]
310	self.imode = self.header_rec['dmode_index'][0]
311	self.antenna = self.header_rec['antenna_coord']
312	self.rx_gains = self.header_rec['rx_gains']
313	self.time = self.header_rec['time'][0]
314	dt = datetime.datetime.utcfromtimestamp(self.time)
315	if dt.date()>self.datatime.date():
316	self.flagDiscontinuousBlock = 1
317	self.datatime = dt
318		254
319	def readData(self):	255	self.header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
		256	self.interval = self.header_rec['interval']
		257	if self.header_rec['magic'] == 888.:
		258	self.header_rec['h0'] = round(self.header_rec['h0'], 2)
		259	nheights = int(self.header_rec['nheights'])
		260	hours = float(self.header_rec['hours'][0])
		261	heights = numpy.arange(nheights) * self.dH + self.header_rec['h0']
		262	datatime = datetime.datetime(self.year, 1, 1) + datetime.timedelta(days=self.doy-1, hours=hours)
		263	return heights, datatime
		264	else:
		265	return False
		266
		267	def readData(self, N):
320	'''	268	'''
321	Reading and filtering data block record of BLTR rawdata file, filtering is according to status_value.	269	Parse data
322
323	Input:
324	status_value - Array data is set to NAN for values that are not equal to status_value
325
326	'''	270	'''
327		271
328	data_structure = numpy.dtype(	272	buffer = numpy.fromfile(self.fp, 'f', 8*N).reshape(N, 8)
329	DATA_STRUCTURE.descr + [	273
330	('rx_saturation', 'u4', (self.nchannels,)),	274	pow0 = buffer[:, 0]
331	('chan_offset', 'u4', (2 * self.nchannels,)),	275	pow1 = buffer[:, 1]
332	('rx_amp', 'u4', (self.nchannels,)),	276	acf0 = (buffer[:,2] + buffer[:,3]*1j) / pow0
333	('rx_snr', 'f4', (self.nchannels,)),	277	acf1 = (buffer[:,4] + buffer[:,5]*1j) / pow1
334	('cross_snr', 'f4', (self.kchan,)),	278	dccf = (buffer[:,6] + buffer[:,7]1j) / (pow0pow1)
335	('sea_power_relative', 'f4', (self.kchan,))]	279
336	)	280	### SNR
337		281	sno = (pow0 + pow1 - self.header_rec['snr']) / self.header_rec['snr']
338	data = numpy.fromfile(self.fp, data_structure, self.nranges)	282	sno10 = numpy.log10(sno)
339		283	# dsno = 1.0 / numpy.sqrt(self.header_file['nint'] * self.header_file['navg']) * (1 + (1 / sno))
340	height = data['range']	284
341	winds = numpy.array(	285	### Vertical Drift
342	(data['zonal'], data['meridional'], data['vertical']))	286	sp = numpy.sqrt(numpy.abs(acf0)*numpy.abs(acf1))
343	snr = data['rx_snr'].T	287	sp[numpy.where(numpy.abs(sp) >= 1.0)] = numpy.sqrt(0.9999)
344		288
345	winds[numpy.where(winds == -9999.)] = numpy.nan	289	vzo = -numpy.arctan2(acf0.imag + acf1.imag,acf0.real + acf1.real)1.5E51.5/(self.ipp*numpy.pi)
346	winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan	290	dvzo = numpy.sqrt(1.0 - spsp)0.3381.5E5/(numpy.sqrt(self.header_file['nint']self.header_file['navg'])spself.ipp)
347	snr[numpy.where(snr == -9999.)] = numpy.nan	291	err = numpy.where(dvzo <= 0.1)
348	snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan	292	dvzo[err] = 0.1
349	snr = numpy.power(10, snr / 10)	293
		294	#Zonal Drifts
		295	dt = self.header_file['nint']*self.ipp / 1.5E5
		296	coh = numpy.sqrt(numpy.abs(dccf))
		297	err = numpy.where(coh >= 1.0)
		298	coh[err] = numpy.sqrt(0.99999)
		299
		300	err = numpy.where(coh <= 0.1)
		301	coh[err] = numpy.sqrt(0.1)
		302
		303	vxo = numpy.arctan2(dccf.imag, dccf.real)self.header_rec['h0']1.0E3/(self.kd*dt)
		304	dvxo = numpy.sqrt(1.0 - cohcoh)self.header_rec['h0']1.0E3/(numpy.sqrt(self.header_file['nint']self.header_file['navg'])cohself.kd*dt)
		305
		306	err = numpy.where(dvxo <= 0.1)
		307	dvxo[err] = 0.1
350		308
351	return height, winds, snr	309	return vzo, dvzo, vxo, dvxo, sno10
352		310
353	def set_output(self):	311	def set_output(self):
354	'''	312	'''
355	Storing data from databuffer to dataOut object	313	Storing data from databuffer to dataOut object
356	'''	314	'''
357		315
358	self.dataOut.data_SNR = self.buffer[0]	316	self.dataOut.data_SNR = self.buffer[4].reshape(1, -1)
359	self.dataOut.heights = self.heights	317	self.dataOut.heightList = self.heights
360	self.dataOut.data_param = self.buffer[1:,]	318	self.dataOut.data_param = self.buffer[0:4,]
361	self.dataOut.utctimeInit = self.time	319	self.dataOut.utctimeInit = self.time
362	self.dataOut.utctime = self.~~dataOut~~.~~utc~~time~~Init~~	320	self.dataOut.utctime = self.time
363	self.dataOut.useLocalTime = ~~Fals~~e	321	self.dataOut.useLocalTime = True
364	self.dataOut.paramInterval = self.~~timedelta~~	322	self.dataOut.paramInterval = self.interval
365	self.dataOut.timezone = self.timezone	323	self.dataOut.timezone = self.timezone
366	self.dataOut.sizeOfFile = self.sizeOfFile	324	self.dataOut.sizeOfFile = self.sizeOfFile
367	self.dataOut.flagNoData = False	325	self.dataOut.flagNoData = False
368	self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock	326	self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock

schainpy/model/utils/jroutils_publish.py +4 -5

                         return self.data[key][self.__times[0]].shape
                     return (0,)
-                def update(self, dataOut):
+                def update(self, dataOut, tm):
                     '''
                     Update data object with new dataOut
                     '''
-                    tm = dataOut.utctime
                     if tm in self.__times:
                         return
                     while True:
                         dataOut = self.receiver.recv_pyobj()
                         if not dataOut.flagNoData:
                             if dataOut.type == 'Parameters':
                                 tm = dataOut.utctimeInit
                             else:
                                 tm = dataOut.utctime
                                 self.data.setup()
                                 self.dates.append(dt)
-                            self.data.update(dataOut)
+                            self.data.update(dataOut, tm)
                         if dataOut.finished is True:
                             self.connections -= 1

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages