schain Commit - r1322:f141c0751dba · Jicamarca Repository

Fix ParametersPlot & BLTRParamReader

Juan C. Espinoza -

r1322:f141c0751dba

parent child

Context file:

r1322:f141c0751dba

Collapse all files

schainpy/model/data/jrodata.py +1 -1

                     for key in self.data:
                         shape = self.shape(key)[:-1] + H.shape
                         for tm, obj in list(self.data[key].items()):
-                            h = self.__heights[self.times.index(tm)]
+                            h = self.__heights[self.times.tolist().index(tm)]
                             if H.size == h.size:
                                 continue
                             index = numpy.where(numpy.in1d(H, h))[0]

schainpy/model/graphics/jroplot_parameters.py +5 0

                     self.ncols = 1
                     self.nrows = self.data.shape(self.CODE)[0]
                     self.nplots = self.nrows
+                    self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95, 'top': 0.95})
+                    if not self.xlabel:
+                        self.xlabel = 'Time'
                     if self.showSNR:
                         self.nrows += 1
                         self.nplots += 1

schainpy/model/io/bltrIO_param.py +69 -121

@@ -16,7 +16,7 import numpy
16	import schainpy.admin	16	import schainpy.admin
17	from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator	17	from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator
18	from schainpy.model.data.jrodata import Parameters	18	from schainpy.model.data.jrodata import Parameters
19	from schainpy.model.io.jroIO_base import ~~JROData~~Reader, ~~isNumber~~	19	from schainpy.model.io.jroIO_base import Reader
20	from schainpy.utils import log	20	from schainpy.utils import log
21		21
22	FILE_HEADER_STRUCTURE = numpy.dtype([	22	FILE_HEADER_STRUCTURE = numpy.dtype([
@@ -85,7 +85,7 DATA_STRUCTURE = numpy.dtype([
85	])	85	])
86		86
87		87
88	class BLTRParamReader(~~JROData~~Reader, ProcessingUnit):	88	class BLTRParamReader(Reader, ProcessingUnit):
89	'''	89	'''
90	Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR	90	Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR
91	from *.sswma files	91	from *.sswma files
@@ -98,143 +98,84 class BLTRParamReader(JRODataReader, ProcessingUnit):
98	ProcessingUnit.__init__(self)	98	ProcessingUnit.__init__(self)
99		99
100	self.dataOut = Parameters()	100	self.dataOut = Parameters()
		101	self.dataOut.timezone = 300
101	self.counter_records = 0	102	self.counter_records = 0
102	self.flagNoMoreFiles = 0	103	self.flagNoMoreFiles = 0
103	self.isConfig = False	104	self.isConfig = False
104	self.filename = None	105	self.filename = None
105		106	self.status_value = 0
106	def setup(self,
107	path=None,
108	startDate=None,
109	endDate=None,
110	ext=None,
111	startTime=datetime.time(0, 0, 0),
112	endTime=datetime.time(23, 59, 59),
113	timezone=0,
114	status_value=0,
115	**kwargs):
116	self.path = path
117	self.startDate = startDate
118	self.endDate = endDate
119	self.startTime = startTime
120	self.endTime = endTime
121	self.status_value = status_value
122	self.datatime = datetime.datetime(1900,1,1)	107	self.datatime = datetime.datetime(1900,1,1)
123	self.delay = kwargs.get('delay', 10)	108	self.filefmt = "*******%Y%m%d****"
124	self.online = kwargs.get('online', False)	109
125	self.nTries = kwargs.get('nTries', 3)	110	def setup(self, **kwargs):
		111
		112	self.set_kwargs(**kwargs)
126		113
127	if self.path is None:	114	if self.path is None:
128	raise ValueError("The path is not valid")	115	raise ValueError("The path is not valid")
129		116
130	if ~~ext~~ is None:	117	if self.online:
131	ext = self.ext	118	log.log("Searching files in online mode...", self.name)
132		119
133	self.fileList = self.search_files(self.path, startDate, endDate, ext)	120	for nTries in range(self.nTries):
134	self.timezone = timezone	121	fullpath = self.searchFilesOnLine(self.path, self.startDate,
135	self.fileIndex = 0	122	self.endDate, self.expLabel, self.ext, self.walk,
		123	self.filefmt, self.folderfmt)
		124	try:
		125	fullpath = next(fullpath)
		126	except:
		127	fullpath = None
		128
		129	if fullpath:
		130	self.fileSize = os.path.getsize(fullpath)
		131	self.filename = fullpath
		132	self.flagIsNewFile = 1
		133	if self.fp != None:
		134	self.fp.close()
		135	self.fp = self.open_file(fullpath, self.open_mode)
		136	self.flagNoMoreFiles = 0
		137	break
136		138
137	if not self.fileList:	139	log.warning(
138	raise Warning("There is no files matching these date in the folder: %s. \n Check 'startDate' and 'endDate' " % (	140	'Waiting {} sec for a valid file in {}: try {} ...'.format(
139	path))	141	self.delay, self.path, nTries + 1),
		142	self.name)
		143	time.sleep(self.delay)
140		144
141	self.setNextFile()	145	if not(fullpath):
		146	raise schainpy.admin.SchainError(
		147	'There isn\'t any valid file in {}'.format(self.path))
		148	self.readFirstHeader()
		149	else:
		150	log.log("Searching files in {}".format(self.path), self.name)
		151	self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
		152	self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
		153	self.setNextFile()
142		154
143	def search_last_file(self):	155	def checkForRealPath(self, nextFile, nextDay):
144	'''	156	'''
145	Get last file and add it to the list
146	'''	157	'''
147		158
148	for n in range(self.nTries+1):	159	dt = self.datatime + datetime.timedelta(1)
149	if n>0:	160	filename = '{}.{}{}'.format(self.siteFile, dt.strftime('%Y%m%d'), self.ext)
150	log.warning(	161	fullfilename = os.path.join(self.path, filename)
151	"Waiting %0.2f seconds for the next file, try %03d ..." % (self.delay, n+1),	162	if os.path.exists(fullfilename):
152	self.name	163	return fullfilename, filename
153	)	164	return None, filename
154	time.sleep(self.delay)	165
155	file_list = os.listdir(self.path)	166
156	file_list.sort()	167	def readFirstHeader(self):
157	if file_list:
158	if self.filename:
159	if file_list[-1] not in self.filename:
160	return file_list[-1]
161	else:
162	continue
163	return file_list[-1]
164	return 0
165
166	def search_files(self, path, startDate, endDate, ext):
167	'''	168	'''
168	Searching for BLTR rawdata file in path
169	Creating a list of file to proces included in [startDate,endDate]
170
171	Input:
172	path - Path to find BLTR rawdata files
173	startDate - Select file from this date
174	enDate - Select file until this date
175	ext - Extension of the file to read
176	'''	169	'''
177
178	log.success('Searching files in {} '.format(path), 'BLTRParamReader')
179	foldercounter = 0
180	fileList0 = glob.glob1(path, "*%s" % ext)
181	fileList0.sort()
182
183	for thisFile in fileList0:
184	year = thisFile[-14:-10]
185	if not isNumber(year):
186	continue
187
188	month = thisFile[-10:-8]
189	if not isNumber(month):
190	continue
191
192	day = thisFile[-8:-6]
193	if not isNumber(day):
194	continue
195		170
196	year, month, day = int(year), int(month), int(day)
197	dateFile = datetime.date(year, month, day)
198
199	if (startDate > dateFile) or (endDate < dateFile):
200	continue
201
202	yield thisFile
203
204	return
205
206	def setNextFile(self):
207
208	if self.online:
209	filename = self.search_last_file()
210	if not filename:
211	self.flagNoMoreFiles = 1
212	return 0
213	else:
214	try:
215	filename = next(self.fileList)
216	except StopIteration:
217	self.flagNoMoreFiles = 1
218	return 0
219
220	log.success('Opening {}'.format(filename), 'BLTRParamReader')
221
222	dirname, name = os.path.split(filename)
223	# 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya	171	# 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
224	self.siteFile = filename.split('.')[0]	172	self.siteFile = self.filename.split('/')[-1].split('.')[0]
225	if self.filename is not None:
226	self.fp.close()
227	self.filename = os.path.join(self.path, filename)
228	self.fp = open(self.filename, 'rb')
229	self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)	173	self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
230	self.nrecords = self.header_file['nrec'][0]	174	self.nrecords = self.header_file['nrec'][0]
231	self.sizeOfFile = os.path.getsize(self.filename)
232	self.counter_records = 0	175	self.counter_records = 0
233	self.flagIsNewFile = 0	176	self.flagIsNewFile = 0
234	self.fileIndex += 1	177	self.fileIndex += 1
235		178
236	return 1
237
238	def readNextBlock(self):	179	def readNextBlock(self):
239		180
240	while True:	181	while True:
@@ -242,7 +183,6 class BLTRParamReader(JRODataReader, ProcessingUnit):
242	self.flagIsNewFile = 1	183	self.flagIsNewFile = 1
243	if not self.setNextFile():	184	if not self.setNextFile():
244	return 0	185	return 0
245
246	try:	186	try:
247	pointer = self.fp.tell()	187	pointer = self.fp.tell()
248	self.readBlock()	188	self.readBlock()
@@ -266,7 +206,6 class BLTRParamReader(JRODataReader, ProcessingUnit):
266		206
267	log.log('Reading Record No. {} -> {}'.format(	207	log.log('Reading Record No. {} -> {}'.format(
268	self.counter_records,	208	self.counter_records,
269	# self.nrecords,
270	self.datatime.ctime()), 'BLTRParamReader')	209	self.datatime.ctime()), 'BLTRParamReader')
271		210
272	return 1	211	return 1
@@ -372,10 +311,8 class BLTRParamReader(JRODataReader, ProcessingUnit):
372	self.dataOut.utctime = self.dataOut.utctimeInit	311	self.dataOut.utctime = self.dataOut.utctimeInit
373	self.dataOut.useLocalTime = False	312	self.dataOut.useLocalTime = False
374	self.dataOut.paramInterval = 157	313	self.dataOut.paramInterval = 157
375	self.dataOut.timezone = self.timezone
376	self.dataOut.site = self.siteFile	314	self.dataOut.site = self.siteFile
377	self.dataOut.nrecords = self.nrecords / self.nmodes	315	self.dataOut.nrecords = self.nrecords / self.nmodes
378	self.dataOut.sizeOfFile = self.sizeOfFile
379	self.dataOut.lat = self.lat	316	self.dataOut.lat = self.lat
380	self.dataOut.lon = self.lon	317	self.dataOut.lon = self.lon
381	self.dataOut.channelList = list(range(self.nchannels))	318	self.dataOut.channelList = list(range(self.nchannels))
@@ -395,13 +332,24 class BLTRParamReader(JRODataReader, ProcessingUnit):
395	'''	332	'''
396	if self.flagNoMoreFiles:	333	if self.flagNoMoreFiles:
397	self.dataOut.flagNoData = True	334	self.dataOut.flagNoData = True
398	raise schainpy.admin.SchainError('No More files to read')	335	return 0
399		336
400	if not self.readNextBlock():	337	if not self.readNextBlock():
401	self.dataOut.flagNoData = True	338	self.dataOut.flagNoData = True
402	raise schainpy.admin.SchainError('Time for wait new file reach!!!')	339	return 0
403		340
404	self.set_output()	341	self.set_output()
405		342
406	return 1	343	return 1
407	No newline at end of file	344
		345	def run(self, **kwargs):
		346	'''
		347	'''
		348
		349	if not(self.isConfig):
		350	self.setup(**kwargs)
		351	self.isConfig = True
		352
		353	self.getData()
		354
		355	return No newline at end of file

schainpy/model/io/jroIO_base.py +30 -28

@@ -528,7 +528,7 class Reader(object):
528	if files:	528	if files:
529	fo = files[-1]	529	fo = files[-1]
530	try:	530	try:
531	dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()	531	dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
532	yield os.path.join(path, expLabel, fo)	532	yield os.path.join(path, expLabel, fo)
533	except Exception as e:	533	except Exception as e:
534	pass	534	pass
@@ -717,6 +717,35 class Reader(object):
717		717
718	pass	718	pass
719		719
		720	def waitDataBlock(self, pointer_location, blocksize=None):
		721	"""
		722	"""
		723
		724	currentPointer = pointer_location
		725	if blocksize is None:
		726	neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
		727	else:
		728	neededSize = blocksize
		729
		730	for nTries in range(self.nTries):
		731	self.fp.close()
		732	self.fp = open(self.filename, 'rb')
		733	self.fp.seek(currentPointer)
		734
		735	self.fileSize = os.path.getsize(self.filename)
		736	currentSize = self.fileSize - currentPointer
		737
		738	if (currentSize >= neededSize):
		739	return 1
		740
		741	log.warning(
		742	"Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1),
		743	self.name
		744	)
		745	time.sleep(self.delay)
		746
		747	return 0
		748
720	class JRODataReader(Reader):	749	class JRODataReader(Reader):
721		750
722	utc = 0	751	utc = 0
@@ -828,33 +857,6 class JRODataReader(Reader):
828		857
829	return 0	858	return 0
830		859
831	def waitDataBlock(self, pointer_location, blocksize=None):
832
833	currentPointer = pointer_location
834	if blocksize is None:
835	neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
836	else:
837	neededSize = blocksize
838
839	for nTries in range(self.nTries):
840	self.fp.close()
841	self.fp = open(self.filename, 'rb')
842	self.fp.seek(currentPointer)
843
844	self.fileSize = os.path.getsize(self.filename)
845	currentSize = self.fileSize - currentPointer
846
847	if (currentSize >= neededSize):
848	return 1
849
850	log.warning(
851	"Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1),
852	self.name
853	)
854	time.sleep(self.delay)
855
856	return 0
857
858	def __setNewBlock(self):	860	def __setNewBlock(self):
859		861
860	if self.fp == None:	862	if self.fp == None:

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages