##// END OF EJS Templates
schain-jc
RSS
test for localtime plotting issues
test for localtime plotting issues
Juan C. Espinoza - - Load All Authors

File last commit:

r1217:f42f79a936d7
r1218:afa054b492c7
Show More
bltrIO_param.py
405 lines | 12.4 KiB | text/x-python | PythonLexer
/ schainpy / model / io / bltrIO_param.py
History | Source | Raw |Copy content |Copy permalink
Juan C. Espinoza
BLTRParamreader ready
 r1010 '''
Created on Nov 9, 2016
@author: roj- LouVD
'''
import os
import sys
import time
import glob
import datetime
Juan C. Espinoza
BLTR ok
 r1018
Juan C. Espinoza
BLTRParamreader ready
 r1010 import numpy
Juan C. Espinoza
Multiprocessing for BLTR (sswma) data
 r1181 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator
Juan C. Espinoza
BLTRParamreader ready
 r1010 from schainpy.model.data.jrodata import Parameters
from schainpy.model.io.jroIO_base import JRODataReader, isNumber
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 from schainpy.utils import log
Juan C. Espinoza
BLTRParamreader ready
 r1010
FILE_HEADER_STRUCTURE = numpy.dtype([
('FMN', '<u4'),
('nrec', '<u4'),
('fr_offset', '<u4'),
('id', '<u4'),
('site', 'u1', (32,))
])
REC_HEADER_STRUCTURE = numpy.dtype([
('rmn', '<u4'),
('rcounter', '<u4'),
('nr_offset', '<u4'),
('tr_offset', '<u4'),
('time', '<u4'),
('time_msec', '<u4'),
('tag', 'u1', (32,)),
('comments', 'u1', (32,)),
('lat', '<f4'),
('lon', '<f4'),
('gps_status', '<u4'),
('freq', '<u4'),
('freq0', '<u4'),
('nchan', '<u4'),
('delta_r', '<u4'),
('nranges', '<u4'),
('r0', '<u4'),
('prf', '<u4'),
('ncoh', '<u4'),
('npoints', '<u4'),
('polarization', '<i4'),
('rx_filter', '<u4'),
('nmodes', '<u4'),
('dmode_index', '<u4'),
('dmode_rngcorr', '<u4'),
('nrxs', '<u4'),
('acf_length', '<u4'),
('acf_lags', '<u4'),
('sea_to_atmos', '<f4'),
('sea_notch', '<u4'),
('lh_sea', '<u4'),
('hh_sea', '<u4'),
('nbins_sea', '<u4'),
('min_snr', '<f4'),
('min_cc', '<f4'),
('max_time_diff', '<f4')
])
DATA_STRUCTURE = numpy.dtype([
('range', '<u4'),
('status', '<u4'),
('zonal', '<f4'),
('meridional', '<f4'),
('vertical', '<f4'),
('zonal_a', '<f4'),
('meridional_a', '<f4'),
('corrected_fading', '<f4'), # seconds
('uncorrected_fading', '<f4'), # seconds
('time_diff', '<f4'),
('major_axis', '<f4'),
('axial_ratio', '<f4'),
('orientation', '<f4'),
('sea_power', '<u4'),
('sea_algorithm', '<u4')
])
Juan C. Espinoza
Multiprocessing for BLTR (sswma) data
 r1181 @MPDecorator
Juan C. Espinoza
BLTRParamreader ready
 r1010 class BLTRParamReader(JRODataReader, ProcessingUnit):
'''
Juan C. Espinoza
Fix reading files in online mode
 r1217 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR
from *.sswma files
Juan C. Espinoza
BLTRParamreader ready
 r1010 '''
ext = '.sswma'
George Yong
Multiprocessing for BLTR (all operations) working
 r1185 def __init__(self):
Juan C. Espinoza
BLTRParamreader ready
 r1010
George Yong
Multiprocessing for BLTR (all operations) working
 r1185 ProcessingUnit.__init__(self)
Juan C. Espinoza
BLTRParamreader ready
 r1010
José Chávez
formatting
 r1083 self.dataOut = Parameters()
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.counter_records = 0
self.flagNoMoreFiles = 0
José Chávez
formatting
 r1083 self.isConfig = False
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.filename = None
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 def setup(self,
path=None,
startDate=None,
endDate=None,
ext=None,
startTime=datetime.time(0, 0, 0),
endTime=datetime.time(23, 59, 59),
timezone=0,
status_value=0,
**kwargs):
self.path = path
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.startDate = startDate
self.endDate = endDate
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.startTime = startTime
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.endTime = endTime
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.status_value = status_value
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.datatime = datetime.datetime(1900,1,1)
Juan C. Espinoza
Add reading online mode for BLTR parameters
 r1215 self.delay = kwargs.get('delay', 10)
self.online = kwargs.get('online', False)
self.nTries = kwargs.get('nTries', 3)
Juan C. Espinoza
BLTRParamreader ready
 r1010
if self.path is None:
George Yong
Python 2to3, Spectra (all operations) working
 r1167 raise ValueError("The path is not valid")
Juan C. Espinoza
BLTRParamreader ready
 r1010
José Chávez
formatting
 r1083 if ext is None:
Juan C. Espinoza
BLTRParamreader ready
 r1010 ext = self.ext
Juan C. Espinoza
Add reading online mode for BLTR parameters
 r1215 self.fileList = self.search_files(self.path, startDate, endDate, ext)
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.timezone = timezone
self.fileIndex = 0
if not self.fileList:
George Yong
Python 2to3, Spectra (all operations) working
 r1167 raise Warning("There is no files matching these date in the folder: %s. \n Check 'startDate' and 'endDate' " % (
path))
Juan C. Espinoza
BLTRParamreader ready
 r1010
self.setNextFile()
José Chávez
formatting
 r1083
Juan C. Espinoza
Add reading online mode for BLTR parameters
 r1215 def search_last_file(self):
'''
Get last file and add it to the list
'''
for n in range(self.nTries):
Juan C. Espinoza
Fix reading files in online mode
 r1217 log.warning(
"Waiting %0.2f seconds for the next file, try %03d ..." % (self.delay, n+1),
self.name
)
time.sleep(self.delay)
Juan C. Espinoza
Add reading online mode for BLTR parameters
 r1215 file_list = os.listdir(self.path)
file_list.sort()
if file_list:
if self.filename:
if file_list[-1] not in self.filename:
return file_list[-1]
else:
continue
return file_list[-1]
return 0
Juan C. Espinoza
BLTRParamreader ready
 r1010 def search_files(self, path, startDate, endDate, ext):
'''
Searching for BLTR rawdata file in path
Creating a list of file to proces included in [startDate,endDate]
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 Input:
path - Path to find BLTR rawdata files
startDate - Select file from this date
enDate - Select file until this date
ext - Extension of the file to read
José Chávez
formatting
 r1083 '''
jespinoza
fixing merge conflicts
 r1090
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 log.success('Searching files in {} '.format(path), 'BLTRParamReader')
Juan C. Espinoza
BLTRParamreader ready
 r1010 foldercounter = 0
fileList0 = glob.glob1(path, "*%s" % ext)
fileList0.sort()
José Chávez
formatting
 r1083
for thisFile in fileList0:
Juan C. Espinoza
BLTRParamreader ready
 r1010 year = thisFile[-14:-10]
if not isNumber(year):
continue
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 month = thisFile[-10:-8]
if not isNumber(month):
continue
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 day = thisFile[-8:-6]
if not isNumber(day):
José Chávez
formatting
 r1083 continue
Juan C. Espinoza
BLTRParamreader ready
 r1010 year, month, day = int(year), int(month), int(day)
dateFile = datetime.date(year, month, day)
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 if (startDate > dateFile) or (endDate < dateFile):
continue
José Chávez
formatting
 r1083
Juan C. Espinoza
Add reading online mode for BLTR parameters
 r1215 yield thisFile
Juan C. Espinoza
BLTRParamreader ready
 r1010
return
def setNextFile(self):
Juan C. Espinoza
Add reading online mode for BLTR parameters
 r1215 if self.online:
filename = self.search_last_file()
if not filename:
self.flagNoMoreFiles = 1
return 0
else:
try:
filename = next(self.fileList)
except StopIteration:
self.flagNoMoreFiles = 1
return 0
log.success('Opening {}'.format(filename), 'BLTRParamReader')
Juan C. Espinoza
BLTRParamreader ready
 r1010
dirname, name = os.path.split(filename)
José Chávez
formatting
 r1083 # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
Juan C. Espinoza
Add reading online mode for BLTR parameters
 r1215 self.siteFile = filename.split('.')[0]
Juan C. Espinoza
BLTRParamreader ready
 r1010 if self.filename is not None:
self.fp.close()
Juan C. Espinoza
Add reading online mode for BLTR parameters
 r1215 self.filename = os.path.join(self.path, filename)
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.fp = open(self.filename, 'rb')
self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
self.nrecords = self.header_file['nrec'][0]
José Chávez
formatting
 r1083 self.sizeOfFile = os.path.getsize(self.filename)
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.counter_records = 0
self.flagIsNewFile = 0
self.fileIndex += 1
return 1
def readNextBlock(self):
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 while True:
Juan C. Espinoza
Fix reading files in online mode
 r1217 if not self.online and self.counter_records == self.nrecords:
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.flagIsNewFile = 1
if not self.setNextFile():
return 0
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085
Juan C. Espinoza
Add reading online mode for BLTR parameters
 r1215 try:
pointer = self.fp.tell()
self.readBlock()
except:
if self.waitDataBlock(pointer, 38512) == 1:
continue
else:
if not self.setNextFile():
return 0
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085
if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
(self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
log.warning(
'Reading Record No. {}/{} -> {} [Skipping]'.format(
self.counter_records,
self.nrecords,
self.datatime.ctime()),
'BLTRParamReader')
Juan C. Espinoza
BLTRParamreader ready
 r1010 continue
break
Juan C. Espinoza
Fix reading files in online mode
 r1217 log.log('Reading Record No. {} -> {}'.format(
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.counter_records,
Juan C. Espinoza
Fix reading files in online mode
 r1217 # self.nrecords,
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.datatime.ctime()), 'BLTRParamReader')
Juan C. Espinoza
BLTRParamreader ready
 r1010
return 1
def readBlock(self):
pointer = self.fp.tell()
header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
Juan C. Espinoza
Multiprocessing for BLTR (sswma) data
 r1181 self.nchannels = int(header_rec['nchan'][0] / 2)
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.kchan = header_rec['nrxs'][0]
self.nmodes = header_rec['nmodes'][0]
self.nranges = header_rec['nranges'][0]
self.fp.seek(pointer)
self.height = numpy.empty((self.nmodes, self.nranges))
George Yong
Multiprocessing for BLTR (all operations) working
 r1185 self.snr = numpy.empty((self.nmodes, int(self.nchannels), self.nranges))
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.buffer = numpy.empty((self.nmodes, 3, self.nranges))
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.flagDiscontinuousBlock = 0
Juan C. Espinoza
BLTRParamreader ready
 r1010
for mode in range(self.nmodes):
José Chávez
formatting
 r1083 self.readHeader()
Juan C. Espinoza
BLTRParamreader ready
 r1010 data = self.readData()
self.height[mode] = (data[0] - self.correction) / 1000.
self.buffer[mode] = data[1]
self.snr[mode] = data[2]
self.counter_records = self.counter_records + self.nmodes
return
def readHeader(self):
'''
RecordHeader of BLTR rawdata file
'''
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 header_structure = numpy.dtype(
REC_HEADER_STRUCTURE.descr + [
George Yong
Multiprocessing for BLTR (all operations) working
 r1185 ('antenna_coord', 'f4', (2, int(self.nchannels))),
('rx_gains', 'u4', (int(self.nchannels),)),
('rx_analysis', 'u4', (int(self.nchannels),))
Juan C. Espinoza
BLTRParamreader ready
 r1010 ]
)
self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
self.lat = self.header_rec['lat'][0]
self.lon = self.header_rec['lon'][0]
self.delta = self.header_rec['delta_r'][0]
self.correction = self.header_rec['dmode_rngcorr'][0]
José Chávez
formatting
 r1083 self.imode = self.header_rec['dmode_index'][0]
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.antenna = self.header_rec['antenna_coord']
self.rx_gains = self.header_rec['rx_gains']
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.time = self.header_rec['time'][0]
dt = datetime.datetime.utcfromtimestamp(self.time)
if dt.date()>self.datatime.date():
self.flagDiscontinuousBlock = 1
self.datatime = dt
Juan C. Espinoza
BLTRParamreader ready
 r1010
def readData(self):
'''
Juan C. Espinoza
Fix reading files in online mode
 r1217 Reading and filtering data block record of BLTR rawdata file,
filtering is according to status_value.
Juan C. Espinoza
BLTRParamreader ready
 r1010
Input:
Juan C. Espinoza
Fix reading files in online mode
 r1217 status_value - Array data is set to NAN for values that are not
equal to status_value
Juan C. Espinoza
BLTRParamreader ready
 r1010
'''
George Yong
Multiprocessing for BLTR (all operations) working
 r1185 self.nchannels = int(self.nchannels)
Juan C. Espinoza
BLTRParamreader ready
 r1010
data_structure = numpy.dtype(
DATA_STRUCTURE.descr + [
('rx_saturation', 'u4', (self.nchannels,)),
('chan_offset', 'u4', (2 * self.nchannels,)),
('rx_amp', 'u4', (self.nchannels,)),
('rx_snr', 'f4', (self.nchannels,)),
('cross_snr', 'f4', (self.kchan,)),
('sea_power_relative', 'f4', (self.kchan,))]
)
data = numpy.fromfile(self.fp, data_structure, self.nranges)
height = data['range']
José Chávez
formatting
 r1083 winds = numpy.array(
(data['zonal'], data['meridional'], data['vertical']))
Juan C. Espinoza
BLTRParamreader ready
 r1010 snr = data['rx_snr'].T
José Chávez
formatting
 r1083 winds[numpy.where(winds == -9999.)] = numpy.nan
Juan C. Espinoza
BLTRParamreader ready
 r1010 winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
snr[numpy.where(snr == -9999.)] = numpy.nan
snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
José Chávez
formatting
 r1083 snr = numpy.power(10, snr / 10)
Juan C. Espinoza
BLTRParamreader ready
 r1010 return height, winds, snr
def set_output(self):
'''
Storing data from databuffer to dataOut object
'''
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.dataOut.data_SNR = self.snr
Juan C. Espinoza
BLTR ok
 r1018 self.dataOut.height = self.height
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.dataOut.data = self.buffer
Juan C. Espinoza
BLTR ok
 r1018 self.dataOut.utctimeInit = self.time
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.dataOut.utctime = self.dataOut.utctimeInit
self.dataOut.useLocalTime = False
self.dataOut.paramInterval = 157
self.dataOut.timezone = self.timezone
self.dataOut.site = self.siteFile
José Chávez
formatting
 r1083 self.dataOut.nrecords = self.nrecords / self.nmodes
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.dataOut.sizeOfFile = self.sizeOfFile
self.dataOut.lat = self.lat
Juan C. Espinoza
BLTR ok
 r1018 self.dataOut.lon = self.lon
George Yong
Python 2to3, Spectra (all operations) working
 r1167 self.dataOut.channelList = list(range(self.nchannels))
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.dataOut.kchan = self.kchan
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.dataOut.delta = self.delta
self.dataOut.correction = self.correction
self.dataOut.nmodes = self.nmodes
self.dataOut.imode = self.imode
self.dataOut.antenna = self.antenna
self.dataOut.rx_gains = self.rx_gains
self.dataOut.flagNoData = False
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
Juan C. Espinoza
BLTRParamreader ready
 r1010
def getData(self):
'''
Storing data from databuffer to dataOut object
'''
if self.flagNoMoreFiles:
self.dataOut.flagNoData = True
Juan C. Espinoza
Review decorator logic for ending process
 r1193 self.dataOut.error = 'No More files to read'
Juan C. Espinoza
Add reading online mode for BLTR parameters
 r1215 return
Juan C. Espinoza
BLTRParamreader ready
 r1010
José Chávez
formatting
 r1083 if not self.readNextBlock():
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.dataOut.flagNoData = True
Juan C. Espinoza
Add reading online mode for BLTR parameters
 r1215 self.dataOut.error = 'Time for wait new file reach!!!'
Juan C. Espinoza
BLTRParamreader ready
 r1010
self.set_output()
George Yong
Multiprocessing for BLTR (all operations) working
 r1185 return 1