##// END OF EJS Templates
schain
RSS
hora y fecha en el primer bloque hdf5 >> consola
hora y fecha en el primer bloque hdf5 >> consola
jespinoza - - Load All Authors

File last commit:

r1343:95b6dc6e7058
r1400:005589ae0527
Show More
bltrIO_param.py
354 lines | 11.0 KiB | text/x-python | PythonLexer
/ schainpy / model / io / bltrIO_param.py
History | Source | Raw |Copy content |Copy permalink
Juan C. Espinoza
BLTRParamreader ready
 r1010 '''
Created on Nov 9, 2016
@author: roj- LouVD
'''
import os
import sys
import time
import glob
import datetime
Juan C. Espinoza
BLTR ok
 r1018
Juan C. Espinoza
BLTRParamreader ready
 r1010 import numpy
jespinoza
Errors handling and gracefully terminate main process
 r1241 import schainpy.admin
Juan C. Espinoza
Multiprocessing for BLTR (sswma) data
 r1181 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator
Juan C. Espinoza
BLTRParamreader ready
 r1010 from schainpy.model.data.jrodata import Parameters
Juan C. Espinoza
Fix ParametersPlot & BLTRParamReader
 r1322 from schainpy.model.io.jroIO_base import Reader
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 from schainpy.utils import log
Juan C. Espinoza
BLTRParamreader ready
 r1010
FILE_HEADER_STRUCTURE = numpy.dtype([
('FMN', '<u4'),
('nrec', '<u4'),
('fr_offset', '<u4'),
('id', '<u4'),
('site', 'u1', (32,))
])
REC_HEADER_STRUCTURE = numpy.dtype([
('rmn', '<u4'),
('rcounter', '<u4'),
('nr_offset', '<u4'),
('tr_offset', '<u4'),
('time', '<u4'),
('time_msec', '<u4'),
('tag', 'u1', (32,)),
('comments', 'u1', (32,)),
('lat', '<f4'),
('lon', '<f4'),
('gps_status', '<u4'),
('freq', '<u4'),
('freq0', '<u4'),
('nchan', '<u4'),
('delta_r', '<u4'),
('nranges', '<u4'),
('r0', '<u4'),
('prf', '<u4'),
('ncoh', '<u4'),
('npoints', '<u4'),
('polarization', '<i4'),
('rx_filter', '<u4'),
('nmodes', '<u4'),
('dmode_index', '<u4'),
('dmode_rngcorr', '<u4'),
('nrxs', '<u4'),
('acf_length', '<u4'),
('acf_lags', '<u4'),
('sea_to_atmos', '<f4'),
('sea_notch', '<u4'),
('lh_sea', '<u4'),
('hh_sea', '<u4'),
('nbins_sea', '<u4'),
('min_snr', '<f4'),
('min_cc', '<f4'),
('max_time_diff', '<f4')
])
DATA_STRUCTURE = numpy.dtype([
('range', '<u4'),
('status', '<u4'),
('zonal', '<f4'),
('meridional', '<f4'),
('vertical', '<f4'),
('zonal_a', '<f4'),
('meridional_a', '<f4'),
('corrected_fading', '<f4'), # seconds
('uncorrected_fading', '<f4'), # seconds
('time_diff', '<f4'),
('major_axis', '<f4'),
('axial_ratio', '<f4'),
('orientation', '<f4'),
('sea_power', '<u4'),
('sea_algorithm', '<u4')
])
Juan C. Espinoza
Rewrite controller, remove MPDecorator to units (keep for plots an writers) use of queues for interproc comm instead of zmq, self operations are no longer supported
 r1287
Juan C. Espinoza
Fix ParametersPlot & BLTRParamReader
 r1322 class BLTRParamReader(Reader, ProcessingUnit):
Juan C. Espinoza
BLTRParamreader ready
 r1010 '''
Juan C. Espinoza
Fix reading files in online mode
 r1217 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR
from *.sswma files
Juan C. Espinoza
BLTRParamreader ready
 r1010 '''
ext = '.sswma'
George Yong
Multiprocessing for BLTR (all operations) working
 r1185 def __init__(self):
Juan C. Espinoza
BLTRParamreader ready
 r1010
George Yong
Multiprocessing for BLTR (all operations) working
 r1185 ProcessingUnit.__init__(self)
Juan C. Espinoza
BLTRParamreader ready
 r1010
José Chávez
formatting
 r1083 self.dataOut = Parameters()
Juan C. Espinoza
Fix ParametersPlot & BLTRParamReader
 r1322 self.dataOut.timezone = 300
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.counter_records = 0
self.flagNoMoreFiles = 0
José Chávez
formatting
 r1083 self.isConfig = False
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.filename = None
Juan C. Espinoza
Fix ParametersPlot & BLTRParamReader
 r1322 self.status_value = 0
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.datatime = datetime.datetime(1900,1,1)
Juan C. Espinoza
Fix ParametersPlot & BLTRParamReader
 r1322 self.filefmt = "*********%Y%m%d******"
def setup(self, **kwargs):
self.set_kwargs(**kwargs)
Juan C. Espinoza
BLTRParamreader ready
 r1010
if self.path is None:
George Yong
Python 2to3, Spectra (all operations) working
 r1167 raise ValueError("The path is not valid")
Juan C. Espinoza
BLTRParamreader ready
 r1010
Juan C. Espinoza
Fix ParametersPlot & BLTRParamReader
 r1322 if self.online:
log.log("Searching files in online mode...", self.name)
for nTries in range(self.nTries):
fullpath = self.searchFilesOnLine(self.path, self.startDate,
self.endDate, self.expLabel, self.ext, self.walk,
self.filefmt, self.folderfmt)
try:
fullpath = next(fullpath)
except:
fullpath = None
if fullpath:
self.fileSize = os.path.getsize(fullpath)
self.filename = fullpath
self.flagIsNewFile = 1
if self.fp != None:
self.fp.close()
self.fp = self.open_file(fullpath, self.open_mode)
self.flagNoMoreFiles = 0
break
Juan C. Espinoza
BLTRParamreader ready
 r1010
Juan C. Espinoza
Fix ParametersPlot & BLTRParamReader
 r1322 log.warning(
'Waiting {} sec for a valid file in {}: try {} ...'.format(
self.delay, self.path, nTries + 1),
self.name)
time.sleep(self.delay)
Juan C. Espinoza
BLTRParamreader ready
 r1010
Juan C. Espinoza
Fix ParametersPlot & BLTRParamReader
 r1322 if not(fullpath):
raise schainpy.admin.SchainError(
'There isn\'t any valid file in {}'.format(self.path))
self.readFirstHeader()
else:
log.log("Searching files in {}".format(self.path), self.name)
self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
self.setNextFile()
José Chávez
formatting
 r1083
Juan C. Espinoza
Fix ParametersPlot & BLTRParamReader
 r1322 def checkForRealPath(self, nextFile, nextDay):
Juan C. Espinoza
Add reading online mode for BLTR parameters
 r1215 '''
'''
Juan C. Espinoza
Fix ParametersPlot & BLTRParamReader
 r1322 dt = self.datatime + datetime.timedelta(1)
filename = '{}.{}{}'.format(self.siteFile, dt.strftime('%Y%m%d'), self.ext)
fullfilename = os.path.join(self.path, filename)
if os.path.exists(fullfilename):
return fullfilename, filename
return None, filename
def readFirstHeader(self):
Juan C. Espinoza
BLTRParamreader ready
 r1010 '''
José Chávez
formatting
 r1083 '''
# 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
Juan C. Espinoza
Fix ParametersPlot & BLTRParamReader
 r1322 self.siteFile = self.filename.split('/')[-1].split('.')[0]
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
self.nrecords = self.header_file['nrec'][0]
self.counter_records = 0
self.flagIsNewFile = 0
Juan C. Espinoza
Add input queues for processing units and external operations
 r1235 self.fileIndex += 1
Juan C. Espinoza
BLTRParamreader ready
 r1010
def readNextBlock(self):
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 while True:
Juan C. Espinoza
Fix reading files in online mode
 r1217 if not self.online and self.counter_records == self.nrecords:
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.flagIsNewFile = 1
if not self.setNextFile():
return 0
Juan C. Espinoza
Add reading online mode for BLTR parameters
 r1215 try:
pointer = self.fp.tell()
self.readBlock()
except:
Juan C. Espinoza
Fix LT issue in plots add snr option in jrodata
 r1219 if self.online and self.waitDataBlock(pointer, 38512) == 1:
Juan C. Espinoza
Add reading online mode for BLTR parameters
 r1215 continue
else:
if not self.setNextFile():
return 0
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085
if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
(self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
log.warning(
'Reading Record No. {}/{} -> {} [Skipping]'.format(
self.counter_records,
self.nrecords,
self.datatime.ctime()),
'BLTRParamReader')
Juan C. Espinoza
BLTRParamreader ready
 r1010 continue
break
Juan C. Espinoza
Fix reading files in online mode
 r1217 log.log('Reading Record No. {} -> {}'.format(
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.counter_records,
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.datatime.ctime()), 'BLTRParamReader')
Juan C. Espinoza
BLTRParamreader ready
 r1010
return 1
def readBlock(self):
pointer = self.fp.tell()
header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
Juan C. Espinoza
Multiprocessing for BLTR (sswma) data
 r1181 self.nchannels = int(header_rec['nchan'][0] / 2)
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.kchan = header_rec['nrxs'][0]
self.nmodes = header_rec['nmodes'][0]
self.nranges = header_rec['nranges'][0]
self.fp.seek(pointer)
self.height = numpy.empty((self.nmodes, self.nranges))
George Yong
Multiprocessing for BLTR (all operations) working
 r1185 self.snr = numpy.empty((self.nmodes, int(self.nchannels), self.nranges))
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.buffer = numpy.empty((self.nmodes, 3, self.nranges))
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.flagDiscontinuousBlock = 0
Juan C. Espinoza
BLTRParamreader ready
 r1010
for mode in range(self.nmodes):
José Chávez
formatting
 r1083 self.readHeader()
Juan C. Espinoza
BLTRParamreader ready
 r1010 data = self.readData()
self.height[mode] = (data[0] - self.correction) / 1000.
self.buffer[mode] = data[1]
self.snr[mode] = data[2]
self.counter_records = self.counter_records + self.nmodes
return
def readHeader(self):
'''
RecordHeader of BLTR rawdata file
'''
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 header_structure = numpy.dtype(
REC_HEADER_STRUCTURE.descr + [
George Yong
Multiprocessing for BLTR (all operations) working
 r1185 ('antenna_coord', 'f4', (2, int(self.nchannels))),
('rx_gains', 'u4', (int(self.nchannels),)),
('rx_analysis', 'u4', (int(self.nchannels),))
Juan C. Espinoza
BLTRParamreader ready
 r1010 ]
)
self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
self.lat = self.header_rec['lat'][0]
self.lon = self.header_rec['lon'][0]
self.delta = self.header_rec['delta_r'][0]
self.correction = self.header_rec['dmode_rngcorr'][0]
José Chávez
formatting
 r1083 self.imode = self.header_rec['dmode_index'][0]
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.antenna = self.header_rec['antenna_coord']
self.rx_gains = self.header_rec['rx_gains']
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.time = self.header_rec['time'][0]
dt = datetime.datetime.utcfromtimestamp(self.time)
if dt.date()>self.datatime.date():
self.flagDiscontinuousBlock = 1
self.datatime = dt
Juan C. Espinoza
BLTRParamreader ready
 r1010
def readData(self):
'''
Juan C. Espinoza
Fix reading files in online mode
 r1217 Reading and filtering data block record of BLTR rawdata file,
filtering is according to status_value.
Juan C. Espinoza
BLTRParamreader ready
 r1010
Input:
Juan C. Espinoza
Fix reading files in online mode
 r1217 status_value - Array data is set to NAN for values that are not
equal to status_value
Juan C. Espinoza
BLTRParamreader ready
 r1010
'''
George Yong
Multiprocessing for BLTR (all operations) working
 r1185 self.nchannels = int(self.nchannels)
Juan C. Espinoza
BLTRParamreader ready
 r1010
data_structure = numpy.dtype(
DATA_STRUCTURE.descr + [
('rx_saturation', 'u4', (self.nchannels,)),
('chan_offset', 'u4', (2 * self.nchannels,)),
('rx_amp', 'u4', (self.nchannels,)),
('rx_snr', 'f4', (self.nchannels,)),
('cross_snr', 'f4', (self.kchan,)),
('sea_power_relative', 'f4', (self.kchan,))]
)
data = numpy.fromfile(self.fp, data_structure, self.nranges)
height = data['range']
José Chávez
formatting
 r1083 winds = numpy.array(
(data['zonal'], data['meridional'], data['vertical']))
Juan C. Espinoza
BLTRParamreader ready
 r1010 snr = data['rx_snr'].T
José Chávez
formatting
 r1083 winds[numpy.where(winds == -9999.)] = numpy.nan
Juan C. Espinoza
BLTRParamreader ready
 r1010 winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
snr[numpy.where(snr == -9999.)] = numpy.nan
snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
José Chávez
formatting
 r1083 snr = numpy.power(10, snr / 10)
Juan C. Espinoza
BLTRParamreader ready
 r1010 return height, winds, snr
def set_output(self):
'''
Storing data from databuffer to dataOut object
'''
José Chávez
formatting
 r1083
jespinoza
Add update method to plots to pass data (no more changes in jrodata)
 r1343 self.dataOut.data_snr = self.snr
Juan C. Espinoza
BLTR ok
 r1018 self.dataOut.height = self.height
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.dataOut.data = self.buffer
Juan C. Espinoza
BLTR ok
 r1018 self.dataOut.utctimeInit = self.time
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.dataOut.utctime = self.dataOut.utctimeInit
self.dataOut.useLocalTime = False
self.dataOut.paramInterval = 157
self.dataOut.site = self.siteFile
José Chávez
formatting
 r1083 self.dataOut.nrecords = self.nrecords / self.nmodes
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.dataOut.lat = self.lat
Juan C. Espinoza
BLTR ok
 r1018 self.dataOut.lon = self.lon
George Yong
Python 2to3, Spectra (all operations) working
 r1167 self.dataOut.channelList = list(range(self.nchannels))
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.dataOut.kchan = self.kchan
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.dataOut.delta = self.delta
self.dataOut.correction = self.correction
self.dataOut.nmodes = self.nmodes
self.dataOut.imode = self.imode
self.dataOut.antenna = self.antenna
self.dataOut.rx_gains = self.rx_gains
self.dataOut.flagNoData = False
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
Juan C. Espinoza
BLTRParamreader ready
 r1010
def getData(self):
'''
Storing data from databuffer to dataOut object
'''
if self.flagNoMoreFiles:
self.dataOut.flagNoData = True
Juan C. Espinoza
Fix ParametersPlot & BLTRParamReader
 r1322 return 0
Juan C. Espinoza
BLTRParamreader ready
 r1010
José Chávez
formatting
 r1083 if not self.readNextBlock():
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.dataOut.flagNoData = True
Juan C. Espinoza
Fix ParametersPlot & BLTRParamReader
 r1322 return 0
Juan C. Espinoza
BLTRParamreader ready
 r1010
self.set_output()
George Yong
Multiprocessing for BLTR (all operations) working
 r1185 return 1
Juan C. Espinoza
Fix ParametersPlot & BLTRParamReader
 r1322
def run(self, **kwargs):
'''
'''
if not(self.isConfig):
self.setup(**kwargs)
self.isConfig = True
self.getData()
return