schain-jc Commit - r1021:2aa9b10a9a70

Operation MAD2Writer done Task #343

Juan C. Espinoza -

r1021:2aa9b10a9a70

parent child

Context file:

r1021:2aa9b10a9a70

Expand all files

schainpy/model/io/__init__.py +2 0

              from jroIO_param import *
              from jroIO_hf import *
+             from jroIO_madrigal import *
              from bltrIO_param import *
              from jroIO_bltr import *
              from jroIO_mira35c import *

schainpy/model/io/bltrIO_param.py +1 -3

                      self.dataOut.data_output = self.buffer
                      self.dataOut.utctimeInit = self.time
                      self.dataOut.utctime = self.dataOut.utctimeInit
-                     self.dataOut.counter_records = self.counter_records
-                     self.dataOut.nrecords = self.nrecords
                      self.dataOut.useLocalTime = False
                      self.dataOut.paramInterval = 157
                      self.dataOut.timezone = self.timezone
                      self.dataOut.site = self.siteFile
-                     self.dataOut.nrecords = self.nrecords
+                     self.dataOut.nrecords = self.nrecords/self.nmodes
                      self.dataOut.sizeOfFile = self.sizeOfFile
                      self.dataOut.lat = self.lat
                      self.dataOut.lon = self.lon

schainpy/model/io/jroIO_madrigal.py +168 -300

		@@ -7,124 +7,146 Created on Aug 1, 2017
7	7	import os
8	8	import sys
9	9	import time
	10	import json
10	11	import datetime
11	12
12	13	import numpy
13	14
14		from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
15		from schainpy.model.data.jrodata import Parameters
16		from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
17		from schainpy.model.graphics.jroplot_parameters import WindProfilerPlot
18		from schainpy.model.io.jroIO_base import *
19
20	15	try:
21	16	import madrigal
22	17	import madrigal.cedar
23		from madrigal.cedar import MadrigalCatalogRecord
24	18	except:
25	19	print 'You should install "madrigal library" module if you want to read/write Madrigal data'
26	20
	21	from schainpy.model.proc.jroproc_base import Operation
	22	from schainpy.model.data.jrodata import Parameters
	23
	24	MISSING = -32767
	25	DEF_CATALOG = {
	26	'principleInvestigator': 'Marco Milla',
	27	'expPurpose': None,
	28	'expMode': None,
	29	'cycleTime': None,
	30	'correlativeExp': None,
	31	'sciRemarks': None,
	32	'instRemarks': None
	33	}
	34	DEF_HEADER = {
	35	'kindatDesc': None,
	36	'analyst': 'Jicamarca User',
	37	'comments': None,
	38	'history': None
	39	}
	40	MNEMONICS = {
	41	10: 'jro',
	42	11: 'jbr',
	43	840: 'jul',
	44	13: 'jas',
	45	1000: 'pbr',
	46	1001: 'hbr',
	47	1002: 'obr',
	48	}
	49
	50	def load_json(obj):
	51	'''
	52	Parse json as string instead of unicode
	53	'''
	54
	55	if isinstance(obj, str):
	56	obj = json.loads(obj)
	57
	58	return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v
	59	for k, v in obj.items()}
	60
27	61
28		class MADWriter(Operation):
	62	class MAD2Writer(Operation):
29	63
30		def __init__(self):
	64	def __init__(self, **kwargs):
31	65
32		Operation.__init__(self)
	66	Operation.__init__(self, **kwargs)
33	67	self.dataOut = Parameters()
34	68	self.path = None
35	69	self.dataOut = None
36		self.flagIsNewFile=1
37		self.ext = ".hdf5"
	70	self.ext = '.dat'
38	71
39	72	return
40	73
41		def run(self, dataOut, path , ~~modetowrite~~,**kwargs):
42
43		if self.flagIsNewFile:
44		flagdata = self.setup(dataOut, path, modetowrite)
	74	def run(self, dataOut, path, oneDList, twoDParam='', twoDList='{}', metadata='{}', **kwargs):
	75	'''
	76	Inputs:
	77	path - path where files will be created
	78	oneDList - json of one-dimensional parameters in record where keys
	79	are Madrigal codes (integers or mnemonics) and values the corresponding
	80	dataOut attribute e.g: {
	81	'gdlatr': 'lat',
	82	'gdlonr': 'lon',
	83	'gdlat2':'lat',
	84	'glon2':'lon'}
	85	twoDParam - independent parameter to get the number of rows e.g:
	86	heighList
	87	twoDList - json of two-dimensional parameters in record where keys
	88	are Madrigal codes (integers or mnemonics) and values the corresponding
	89	dataOut attribute if multidimensional array specify as tupple
	90	('attr', pos) e.g: {
	91	'gdalt': 'heightList',
	92	'vn1p2': ('data_output', 0),
	93	'vn2p2': ('data_output', 1),
	94	'vn3': ('data_output', 2),
	95	'snl': ('data_SNR', 'db')
	96	}
	97	metadata - json of madrigal metadata (kinst, kindat, catalog and header)
	98	'''
	99	if not self.isConfig:
	100	self.setup(dataOut, path, oneDList, twoDParam, twoDList, metadata, **kwargs)
	101	self.isConfig = True
45	102
46	103	self.putData()
47	104	return
48	105
49		def setup(self, dataOut, path, ~~modetowrite~~):
	106	def setup(self, dataOut, path, oneDList, twoDParam, twoDList, metadata, **kwargs):
50	107	'''
51		Recovering data to write in new *.hdf5 file
52		Inputs:
53		modew -- mode to write (1 or 2)
54		path -- destination path
55
	108	Configure Operation
56	109	'''
57	110
58		self.im = modetowrite-1
59		if self.im!=0 and self.im!=1:
60		raise ValueError, 'Check "modetowrite" value. Must be egual to 1 or 2, "{}" is not valid. '.format(modetowrite)
61
62	111	self.dataOut = dataOut
63		self.nmodes = self.dataOut.nmodes
64		self.nchannels = self.dataOut.nchannels
65		self.lat = self.dataOut.lat
66		self.lon = self.dataOut.lon
67		self.hcm = 3
68		self.thisDate = self.dataOut.utctimeInit
69		self.year = self.dataOut.year
70		self.month = self.dataOut.month
71		self.day = self.dataOut.day
	112	self.nmodes = self.dataOut.nmodes
72	113	self.path = path
73
74		self.flagIsNewFile = 0
75
76		return 1
77
	114	self.blocks = kwargs.get('blocks', None)
	115	self.counter = 0
	116	self.oneDList = load_json(oneDList)
	117	self.twoDList = load_json(twoDList)
	118	self.twoDParam = twoDParam
	119	meta = load_json(metadata)
	120	self.kinst = meta.get('kinst')
	121	self.kindat = meta.get('kindat')
	122	self.catalog = meta.get('catalog', DEF_CATALOG)
	123	self.header = meta.get('header', DEF_HEADER)
	124
	125	return
	126
78	127	def setFile(self):
79	128	'''
80		- Determining the file name for each mode of operation
81		kinst - Kind of Instrument (mnemotic)
82		kindat - Kind of Data (mnemotic)
83
84		- Creating a cedarObject
85
86		'''
87		lat_piura = -5.17
88		lat_huancayo = -12.04
89		lat_porcuya = -5.8
90
91		if '%2.2f' % self.lat == '%2.2f' % lat_piura:
92		self.instMnemonic = 'pbr'
93
94		elif '%2.2f' % self.lat == '%2.2f' % lat_huancayo:
95		self.instMnemonic = 'hbr'
96
97		elif '%2.2f' % self.lat == '%2.2f' % lat_porcuya:
98		self.instMnemonic = 'obr'
99		else: raise Warning, "The site of file read doesn't match any site known. Only file from Huancayo, Piura and Porcuya can be processed.\n Check the file "
100
101		mode = ['_mode1','_mode2']
102
103		self.hdf5filename = '%s%4.4d%2.2d%2.2d%s%s' % (self.instMnemonic,
104		self.year,
105		self.month,
106		self.day,
107		mode[self.im],
108		self.ext)
	129	Create new cedar file object
	130	'''
	131
	132	self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
	133	date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
	134
	135	filename = '%s%s_%s%s' % (self.mnemonic,
	136	date.strftime('%Y%m%d_%H%M%S'),
	137	self.dataOut.mode,
	138	self.ext)
109	139
110		self.fullname=os.path.join(self.path,~~self~~.~~hdf5~~filename)
	140	self.fullname = os.path.join(self.path, filename)
111	141
112	142	if os.path.isfile(self.fullname) :
113	143	print "Destination path '%s' already exists. Previous file deleted. " %self.fullname
114	144	os.remove(self.fullname)
115
116		# Identify kinst and kindat
117		InstName = self.hdf5filename[0:3]
118		KinstList = [1000, 1001, 1002]
119		KinstId = {'pbr':0, 'hbr':1, 'obr':2} # pbr:piura, hbr:huancayo, obr:porcuya
120		KindatList = [1600, 1601] # mode 1, mode 2
121		self.type = KinstId[InstName]
122		self.kinst = KinstList[self.type]
123		self.kindat = KindatList[self.im]
124	145
125	146	try:
	147	print '[Writing] creating file : %s' % (self.fullname)
126	148	self.cedarObj = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
127		except ValueError, ~~messag~~e:
	149	except ValueError, e:
128	150	print '[Error]: Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile" '
129	151	return
130	152
		@@ -132,244 +154,90 class MADWriter(Operation):
132	154
133	155	def writeBlock(self):
134	156	'''
135		- Selecting mode of operation:
136
137		bltr high resolution mode 1 - Low Atmosphere (0 - 3km) // bltr high resolution mode 2 - High Atmosphere (0 - 10km)
138		msnr - Average Signal Noise Ratio in dB
139		hcm - 3 km
140
141		- Filling the cedarObject by a block: each array data entry is assigned a code that defines the parameter to write to the file
142
143		GDLATR - Reference geod latitude (deg)
144		GDLONR - Reference geographic longitude (deg)
145		GDLAT2 - Geodetic latitude of second inst (deg)
146		GLON2 - Geographic longitude of second inst (deg)
147
148		GDALT - Geodetic altitude (height) (km)
149		SNL - Log10 (signal to noise ratio)
150		VN1P2 - Neutral wind in direction 1 (eastward) (m/s), ie zonal wind
151		VN2P2 - Neutral wind in direction 2 (northward) (m/s), ie meridional wind
152		EL2 - Ending elevation angle (deg), ie vertical wind
153
154		Other parameters: /madrigal3/metadata/parcodes.tab
155
	157	Add data records to cedar file taking data from oneDList and twoDList
	158	attributes.
	159	Allowed parameters in: parcodes.tab
156	160	'''
157	161
158		self.z_zon = self.dataOut.data_output[0,:,:]
159		self.z_mer =self.dataOut.data_output[1,:,:]
160		self.z_ver = self.dataOut.data_output[2,:,:]
161
162		if self.im == 0:
163		h_select = numpy.where(numpy.bitwise_and(self.dataOut.height[0, :] >= 0., self.dataOut.height[0, :] <= self.hcm, numpy.isfinite(self.dataOut.height[0, :])))
164		else:
165		h_select = numpy.where(numpy.bitwise_and(self.dataOut.height[0, :] >= 0., self.dataOut.height[0, :] < 20, numpy.isfinite(self.dataOut.height[0, :])))
166
167		ht = h_select[0]
168
169		self.o_height = self.dataOut.height[self.im, ht]
170		self.o_zon = self.z_zon[ht, self.im]
171		self.o_mer = self.z_mer[ht, self.im]
172		self.o_ver = self.z_ver[ht, self.im]
173		o_snr = self.dataOut.data_SNR[ :, :, self.im]
174
175		o_snr = o_snr[ht, :]
176
177		ndiv = numpy.nansum((numpy.isfinite(o_snr)), 1)
178		ndiv = ndiv.astype(float)
179
180		sel_div = numpy.where(ndiv == 0.)
181		ndiv[sel_div] = numpy.nan
182
183		if self.nchannels > 1:
184		msnr = numpy.nansum(o_snr, axis=1)
185		else:
186		msnr = o_snr
187
188		try:
189		self.msnr = 10 * numpy.log10(msnr / ndiv)
190		except ZeroDivisionError:
191		self.msnr = 10 * numpy.log10(msnr /1)
192		print 'Number of division (ndiv) egal to 1 by default. Check SNR'
193
194		time_t = time.gmtime(self.dataOut.time1)
195		year = time_t.tm_year
196		month = time_t.tm_mon
197		day = time_t.tm_mday
198		hour = time_t.tm_hour
199		minute = time_t.tm_min
200		second = time_t.tm_sec
201		timedate_0 = datetime.datetime(year, month, day, hour, minute, second)
202
203		# 1d parameters
204		GDLATR = self.lat
205		GDLONR = self.lon
206		GDLAT2 = self.lat
207		GLON2 = self.lon
208
209		# 2d parameters
210		GDALT = self.o_height
211
212		SNL = self.msnr
213		VN1P2 = self.o_zon
214		VN2P2 = self.o_mer
215		EL2 = self.o_ver
216		NROW = len(self.o_height)
217
218		startTime = timedate_0
219		endTime = startTime
220		self.dataRec = madrigal.cedar.MadrigalDataRecord(self.kinst,
221		self.kindat,
222		startTime.year,
223		startTime.month,
224		startTime.day,
225		startTime.hour,
226		startTime.minute,
227		startTime.second,
228		0,
229		endTime.year,
230		endTime.month,
231		endTime.day,
232		endTime.hour,
233		endTime.minute,
234		endTime.second,
235		0,
236		('gdlatr', 'gdlonr', 'gdlat2', 'glon2'),
237		('gdalt', 'snl', 'vn1p2', 'vn2p2', 'el2'),
238		NROW, ind2DList=['gdalt'])
	162	startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
	163	endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
	164	nrows = len(getattr(self.dataOut, self.twoDParam))
	165
	166	rec = madrigal.cedar.MadrigalDataRecord(
	167	self.kinst,
	168	self.kindat,
	169	startTime.year,
	170	startTime.month,
	171	startTime.day,
	172	startTime.hour,
	173	startTime.minute,
	174	startTime.second,
	175	startTime.microsecond/10000,
	176	endTime.year,
	177	endTime.month,
	178	endTime.day,
	179	endTime.hour,
	180	endTime.minute,
	181	endTime.second,
	182	endTime.microsecond/10000,
	183	self.oneDList.keys(),
	184	self.twoDList.keys(),
	185	nrows
	186	)
239	187
240		# Setting 1d values
241		self.dataRec.set1D('gdlatr', GDLATR)
242		self.dataRec.set1D('gdlonr', GDLONR)
243		self.dataRec.set1D('gdlat2', GDLAT2)
244		self.dataRec.set1D('glon2', GLON2)
	188	# Setting 1d values
	189	for key in self.oneDList:
	190	rec.set1D(key, getattr(self.dataOut, self.oneDList[key]))
245	191
246	192	# Setting 2d values
247		for n in range(self.o_height.shape[0]):
248		self.dataRec.set2D('gdalt', n, GDALT[n])
249		self.dataRec.set2D('snl', n, SNL[n])
250		self.dataRec.set2D('vn1p2', n, VN1P2[n])
251		self.dataRec.set2D('vn2p2', n, VN2P2[n])
252		self.dataRec.set2D('el2', n, EL2[n])
253
254		# Appending new data record
255		'''
256		[MADRIGAL3]There are two ways to write to a MadrigalCedarFile. Either this method (write) is called after all the
257		records have been appended to the MadrigalCedarFile, or dump is called after a certain number of records are appended,
258		and then at the end dump is called a final time if there were any records not yet dumped, followed by addArray.
259		'''
260
261		self.cedarObj.append(self.dataRec)
262		print ' [Writing] records {} (mode {}).'.format(self.dataOut.counter_records,self.im+1)
	193	invalid = numpy.isnan(self.dataOut.data_output)
	194	self.dataOut.data_output[invalid] = MISSING
	195	out = {}
	196	for key, value in self.twoDList.items():
	197	if isinstance(value, str):
	198	out[key] = getattr(self.dataOut, value)
	199	elif isinstance(value, tuple):
	200	attr, x = value
	201	if isinstance(x, (int, float)):
	202	out[key] = getattr(self.dataOut, attr)[int(x)]
	203	elif x.lower()=='db':
	204	tmp = getattr(self.dataOut, attr)
	205	SNRavg = numpy.average(tmp, axis=0)
	206	out[key] = 10*numpy.log10(SNRavg)
	207
	208	for n in range(nrows):
	209	for key in out:
	210	rec.set2D(key, n, out[key][n])
	211
	212	self.cedarObj.append(rec)
263	213	self.cedarObj.dump()
264
265
	214	print '[Writing] Record No. {} (mode {}).'.format(
	215	self.counter,
	216	self.dataOut.mode
	217	)
266	218
267
268	219	def setHeader(self):
269	220	'''
270		- Creating self.catHeadObj
271		- Adding information catalog
272		- Writing file header
273
	221	Create an add catalog and header to cedar file
274	222	'''
275		self.catHeadObj = madrigal.cedar.CatalogHeaderCreator(self.fullname)
276		kindatDesc, comments, analyst, history, principleInvestigator = self._info_BLTR()
277
278		self.catHeadObj.createCatalog(principleInvestigator="Jarjar",
279		expPurpose='characterize the atmospheric dynamics in this region where frequently it happens the El Nino',
280		sciRemarks="http://madrigal3.haystack.mit.edu/static/CEDARMadrigalHdf5Format.pdf")
281
282		self.catHeadObj.createHeader(kindatDesc, analyst, comments, history)
283
284		self.catHeadObj.write()
285	223
286		print '[File created] path: %s' % (self.fullname)
	224	header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
	225	header.createCatalog(**self.catalog)
	226	header.createHeader(**self.header)
	227	header.write()
287	228
288	229	def putData(self):
289	230
290	231	if self.dataOut.flagNoData:
291	232	return 0
292	233
293		if self.~~dataOut~~.counter~~_records~~ == 1:
294		self.setFile()
295		print '[Writing] Setting new hdf5 file for the mode {}'.format(self.im+1)
	234	if self.counter == 0:
	235	self.setFile()
296	236
297		if self.~~dataOut~~.counter~~_records~~ <= self.dataOut.nrecords:
	237	if self.counter <= self.dataOut.nrecords:
298	238	self.writeBlock()
299
	239	self.counter += 1
300	240
301		if self.~~dataOut~~.counter~~_records~~ == self.dataOut.nrecords:
302		self.cedarObj.addArray()
303
	241	if self.counter == self.dataOut.nrecords or self.counter == self.blocks:
304	242	self.setHeader()
305		self.~~flagIsNewFile~~ = 1
306
307		def _info_BLTR(self):
308
309		kindatDesc = '''--This header is for KINDAT = %d''' % self.kindat
310		history = None
311		analyst = '''Jarjar'''
312		principleInvestigator = '''
313		Jarjar
314		Radio Observatorio de Jicamarca
315		Instituto Geofisico del Peru
316
317		'''
318		if self.type == 1:
319		comments = '''
320
321		--These data are provided by two Boundary Layer and Tropospheric Radar (BLTR) deployed at two different locations at Peru(GMT-5), one of them at Piura(5.17 S, 80.64W) and another located at Huancayo (12.04 S, 75.32 W).
322
323		--The purpose of conducting these observations is to measure wind in the differents levels of height, this radar makes measurements the Zonal(U), Meridional(V) and Vertical(W) wind velocities component in northcoast from Peru. And the main purpose of these mensurations is to characterize the atmospheric dynamics in this region where frequently it happens the 'El Nino Phenomenon'
324
325		--In Kindat = 1600, contains information of wind velocities component since 0 Km to 3 Km.
326
327		--In Kindat = 1601, contains information of wind velocities component since 0 Km to 10 Km.
328
329		--The Huancayo-BLTR is a VHF Profiler Radar System is a 3 channel coherent receiver pulsed radar utilising state-of-the-art software and computing techniques to acquire, decode, and translate signals obtained from partial reflection echoes in the troposphere, lower stratosphere and mesosphere. It uses an array of three horizontal spaced and vertically directed receiving antennas. The data is recorded thirty seconds, averaged to one minute mean values of Height, Zonal, Meridional and Vertical wind.
330
331		--The Huancayo-BLTR was installed in January 2010. This instrument was designed and constructed by Genesis Soft Pty. Ltd. Is constituted by three groups of spaced antennas (distributed) forming an isosceles triangle.
332
333
334		Station _______ Geographic Coord ______ Geomagnetic Coord
335
336		_______________ Latitude _ Longitude __ Latitude _ Longitude
337
338		Huancayo (HUA) __12.04 S ___ 75.32 W _____ -12.05 ____ 352.85
339		Piura (PIU) _____ 5.17 S ___ 80.64 W ______ 5.18 ____ 350.93
340
341		WIND OBSERVATIONS
342
343		--To obtain wind the BLTR uses Spaced Antenna technique (e.g., Briggs 1984). The scatter and reflection it still provided by variations in the refractive index as in the Doppler method(Gage and Basley,1978; Balsley and Gage 1982; Larsen and Rottger 1982), but instead of using the Doppler shift to derive the velocity components, the cross-correlation between signals in an array of three horizontally spaced and vertically directed receiving antennas is used.
344
345		......................................................................
346		For more information, consult the following references:
347		- Balsley, B. B., and K. S. Gage., On the use of radars for operational wind profiling, Bull. Amer. Meteor.Soc.,63, 1009-1018, 1982.
348
349		- Briggs, B. H., The analysis of spaced sensor data by correations techniques, Handbook for MAP, Vol. 13, SCOTEP Secretariat, University of Illinois, Urbana, 166-186, 1984.
350
351		- Gage, K. S., and B.B. Balsley., Doppler radar probing of the clear atmosphere, Bull. Amer. Meteor.Soc., 59, 1074-1093, 1978.
352
353		- Larsen, M. F., The Spaced Antenna Technique for Radar Wind Profiling, Journal of Atm. and Ocean. Technology. , Vol.6, 920-937, 1989.
354
355		- Larsen, M. F., A method for single radar voracity measurements?, Handbook for MAP,SCOSTEP Secretariat, University of the Illinois, Urban, in press, 1989.
356		......................................................................
357
358		ACKNOWLEDGEMENTS:
359
360		--The Piura and Huancayo BLTR are part of the network of instruments operated by the Jicamarca Radio Observatory.
361
362		--The Jicamarca Radio Observatory is a facility of the Instituto Geofisico del Peru operated with support from the NSF Cooperative Agreement ATM-0432565 through Cornell University
363
364		......................................................................
365
366		Further questions and comments should be addressed to:
367		Radio Observatorio de Jicamarca
368		Instituto Geofisico del Peru
369		Lima, Peru
370		Web URL: http://jro.igp.gob.pe
371		......................................................................
372		'''
373
374		return kindatDesc, comments, analyst, history, principleInvestigator
375
	243	self.counter = 0

schainpy/model/proc/bltrproc_parameters.py +11 -1

                      '''
                      ProcessingUnit.__init__(self, **kwargs)
                      self.dataOut = Parameters()
+                     self.isConfig = False
-                 def run(self, mode, snr_threshold=None):
+                 def setup(self, mode):
+                     '''
-                     '''
+                     self.dataOut.mode = mode
+                 def run(self, mode, snr_threshold=None):
+                     '''
                      Inputs:
                          mode = High resolution (0) or Low resolution (1) data
                          snr_threshold = snr filter value
                      '''
+                     if not self.isConfig:
+                         self.setup(mode)
+                         self.isConfig = True
                      if self.dataIn.type == 'Parameters':
                          self.dataOut.copy(self.dataIn)

schainpy/__init__.pyc removed binary 0 0