##// END OF EJS Templates
schain
RSS
Se realizar la lectura en modo online llamando al metodo digitalRFReader(self.path) en reemplazo del metodo reload(), grabando previamente el path de lectura o directorio superior donde se almacena la data. Adicionalmente, se ha definido un tiempo de espera de 3 segundos para dar tiempo suficiente al programa de adquisicion de generar archivos. ...
Se realizar la lectura en modo online llamando al metodo digitalRFReader(self.path) en reemplazo del metodo reload(), grabando previamente el path de lectura o directorio superior donde se almacena la data. Adicionalmente, se ha definido un tiempo de espera de 3 segundos para dar tiempo suficiente al programa de adquisicion de generar archivos. El archivo jroIO_digitalRF.py utiliza la libreria digital_rf cuya version actual es la 2.62( 2017 ) ,esta libreria no tiene definido el metodo o clase reload, este metodo existe en la version 2.0(2014), si uno revisa el archivo jroIO_usrp.py, esta unidad de lectura trabaja con la version 2.0 llamada digital_rf_hdf5, para hacer uso de esta unidad de lectura se instalan los programas correspondiente pero el formato y la informacion difiere un poco de la version actual. Se infiere entonces que al desarrollar del archivo jroIO_digitalRF.py, esperaba que la libreria aun tenga incluido el metodo reload con el update de las versiones pero este ya no es parte del desarrollo, Se realizo la consulta al desarrollador actual de digitalRF Ryan Voltz si se iba a incluir a futuro pero indico que no era necesario.
jespinoza - - Load All Authors

File last commit:

r1090:36f30f86830d merge
r1234:b6a76136b1f3
Show More
bltrIO_param.py
369 lines | 11.2 KiB | text/x-python | PythonLexer
/ schainpy / model / io / bltrIO_param.py
History | Source | Raw |Copy content |Copy permalink
Juan C. Espinoza
BLTRParamreader ready
 r1010 '''
Created on Nov 9, 2016
@author: roj- LouVD
'''
import os
import sys
import time
import glob
import datetime
Juan C. Espinoza
BLTR ok
 r1018
Juan C. Espinoza
BLTRParamreader ready
 r1010 import numpy
from schainpy.model.proc.jroproc_base import ProcessingUnit
from schainpy.model.data.jrodata import Parameters
from schainpy.model.io.jroIO_base import JRODataReader, isNumber
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 from schainpy.utils import log
Juan C. Espinoza
BLTRParamreader ready
 r1010
FILE_HEADER_STRUCTURE = numpy.dtype([
('FMN', '<u4'),
('nrec', '<u4'),
('fr_offset', '<u4'),
('id', '<u4'),
('site', 'u1', (32,))
])
REC_HEADER_STRUCTURE = numpy.dtype([
('rmn', '<u4'),
('rcounter', '<u4'),
('nr_offset', '<u4'),
('tr_offset', '<u4'),
('time', '<u4'),
('time_msec', '<u4'),
('tag', 'u1', (32,)),
('comments', 'u1', (32,)),
('lat', '<f4'),
('lon', '<f4'),
('gps_status', '<u4'),
('freq', '<u4'),
('freq0', '<u4'),
('nchan', '<u4'),
('delta_r', '<u4'),
('nranges', '<u4'),
('r0', '<u4'),
('prf', '<u4'),
('ncoh', '<u4'),
('npoints', '<u4'),
('polarization', '<i4'),
('rx_filter', '<u4'),
('nmodes', '<u4'),
('dmode_index', '<u4'),
('dmode_rngcorr', '<u4'),
('nrxs', '<u4'),
('acf_length', '<u4'),
('acf_lags', '<u4'),
('sea_to_atmos', '<f4'),
('sea_notch', '<u4'),
('lh_sea', '<u4'),
('hh_sea', '<u4'),
('nbins_sea', '<u4'),
('min_snr', '<f4'),
('min_cc', '<f4'),
('max_time_diff', '<f4')
])
DATA_STRUCTURE = numpy.dtype([
('range', '<u4'),
('status', '<u4'),
('zonal', '<f4'),
('meridional', '<f4'),
('vertical', '<f4'),
('zonal_a', '<f4'),
('meridional_a', '<f4'),
('corrected_fading', '<f4'), # seconds
('uncorrected_fading', '<f4'), # seconds
('time_diff', '<f4'),
('major_axis', '<f4'),
('axial_ratio', '<f4'),
('orientation', '<f4'),
('sea_power', '<u4'),
('sea_algorithm', '<u4')
])
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 class BLTRParamReader(JRODataReader, ProcessingUnit):
'''
Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR from *.sswma files
'''
ext = '.sswma'
def __init__(self, **kwargs):
José Chávez
formatting
 r1083 ProcessingUnit.__init__(self, **kwargs)
Juan C. Espinoza
BLTRParamreader ready
 r1010
José Chávez
formatting
 r1083 self.dataOut = Parameters()
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.counter_records = 0
self.flagNoMoreFiles = 0
José Chávez
formatting
 r1083 self.isConfig = False
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.filename = None
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 def setup(self,
path=None,
startDate=None,
endDate=None,
ext=None,
startTime=datetime.time(0, 0, 0),
endTime=datetime.time(23, 59, 59),
timezone=0,
status_value=0,
**kwargs):
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.path = path
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.startDate = startDate
self.endDate = endDate
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.startTime = startTime
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.endTime = endTime
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.status_value = status_value
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.datatime = datetime.datetime(1900,1,1)
Juan C. Espinoza
BLTRParamreader ready
 r1010
if self.path is None:
raise ValueError, "The path is not valid"
José Chávez
formatting
 r1083 if ext is None:
Juan C. Espinoza
BLTRParamreader ready
 r1010 ext = self.ext
self.search_files(self.path, startDate, endDate, ext)
self.timezone = timezone
self.fileIndex = 0
if not self.fileList:
José Chávez
formatting
 r1083 raise Warning, "There is no files matching these date in the folder: %s. \n Check 'startDate' and 'endDate' " % (
path)
Juan C. Espinoza
BLTRParamreader ready
 r1010
self.setNextFile()
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 def search_files(self, path, startDate, endDate, ext):
'''
Searching for BLTR rawdata file in path
Creating a list of file to proces included in [startDate,endDate]
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 Input:
path - Path to find BLTR rawdata files
startDate - Select file from this date
enDate - Select file until this date
ext - Extension of the file to read
José Chávez
formatting
 r1083 '''
jespinoza
fixing merge conflicts
 r1090
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 log.success('Searching files in {} '.format(path), 'BLTRParamReader')
Juan C. Espinoza
BLTRParamreader ready
 r1010 foldercounter = 0
fileList0 = glob.glob1(path, "*%s" % ext)
fileList0.sort()
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.fileList = []
self.dateFileList = []
José Chávez
formatting
 r1083
for thisFile in fileList0:
Juan C. Espinoza
BLTRParamreader ready
 r1010 year = thisFile[-14:-10]
if not isNumber(year):
continue
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 month = thisFile[-10:-8]
if not isNumber(month):
continue
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 day = thisFile[-8:-6]
if not isNumber(day):
José Chávez
formatting
 r1083 continue
Juan C. Espinoza
BLTRParamreader ready
 r1010 year, month, day = int(year), int(month), int(day)
dateFile = datetime.date(year, month, day)
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 if (startDate > dateFile) or (endDate < dateFile):
continue
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.fileList.append(thisFile)
self.dateFileList.append(dateFile)
return
def setNextFile(self):
file_id = self.fileIndex
if file_id == len(self.fileList):
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 log.success('No more files in the folder', 'BLTRParamReader')
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.flagNoMoreFiles = 1
return 0
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 log.success('Opening {}'.format(self.fileList[file_id]), 'BLTRParamReader')
Juan C. Espinoza
BLTRParamreader ready
 r1010 filename = os.path.join(self.path, self.fileList[file_id])
dirname, name = os.path.split(filename)
José Chávez
formatting
 r1083 # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
self.siteFile = name.split('.')[0]
Juan C. Espinoza
BLTRParamreader ready
 r1010 if self.filename is not None:
self.fp.close()
self.filename = filename
self.fp = open(self.filename, 'rb')
self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
self.nrecords = self.header_file['nrec'][0]
José Chávez
formatting
 r1083 self.sizeOfFile = os.path.getsize(self.filename)
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.counter_records = 0
self.flagIsNewFile = 0
self.fileIndex += 1
return 1
def readNextBlock(self):
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 while True:
Juan C. Espinoza
BLTRParamreader ready
 r1010 if self.counter_records == self.nrecords:
self.flagIsNewFile = 1
if not self.setNextFile():
return 0
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.readBlock()
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085
if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
(self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
log.warning(
'Reading Record No. {}/{} -> {} [Skipping]'.format(
self.counter_records,
self.nrecords,
self.datatime.ctime()),
'BLTRParamReader')
Juan C. Espinoza
BLTRParamreader ready
 r1010 continue
break
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 log.log('Reading Record No. {}/{} -> {}'.format(
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.counter_records,
self.nrecords,
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.datatime.ctime()), 'BLTRParamReader')
Juan C. Espinoza
BLTRParamreader ready
 r1010
return 1
def readBlock(self):
pointer = self.fp.tell()
header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
José Chávez
formatting
 r1083 self.nchannels = header_rec['nchan'][0] / 2
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.kchan = header_rec['nrxs'][0]
self.nmodes = header_rec['nmodes'][0]
self.nranges = header_rec['nranges'][0]
self.fp.seek(pointer)
self.height = numpy.empty((self.nmodes, self.nranges))
self.snr = numpy.empty((self.nmodes, self.nchannels, self.nranges))
self.buffer = numpy.empty((self.nmodes, 3, self.nranges))
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.flagDiscontinuousBlock = 0
Juan C. Espinoza
BLTRParamreader ready
 r1010
for mode in range(self.nmodes):
José Chávez
formatting
 r1083 self.readHeader()
Juan C. Espinoza
BLTRParamreader ready
 r1010 data = self.readData()
self.height[mode] = (data[0] - self.correction) / 1000.
self.buffer[mode] = data[1]
self.snr[mode] = data[2]
self.counter_records = self.counter_records + self.nmodes
return
def readHeader(self):
'''
RecordHeader of BLTR rawdata file
'''
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 header_structure = numpy.dtype(
REC_HEADER_STRUCTURE.descr + [
('antenna_coord', 'f4', (2, self.nchannels)),
('rx_gains', 'u4', (self.nchannels,)),
('rx_analysis', 'u4', (self.nchannels,))
]
)
self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
self.lat = self.header_rec['lat'][0]
self.lon = self.header_rec['lon'][0]
self.delta = self.header_rec['delta_r'][0]
self.correction = self.header_rec['dmode_rngcorr'][0]
José Chávez
formatting
 r1083 self.imode = self.header_rec['dmode_index'][0]
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.antenna = self.header_rec['antenna_coord']
self.rx_gains = self.header_rec['rx_gains']
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.time = self.header_rec['time'][0]
dt = datetime.datetime.utcfromtimestamp(self.time)
if dt.date()>self.datatime.date():
self.flagDiscontinuousBlock = 1
self.datatime = dt
Juan C. Espinoza
BLTRParamreader ready
 r1010
def readData(self):
'''
Reading and filtering data block record of BLTR rawdata file, filtering is according to status_value.
Input:
status_value - Array data is set to NAN for values that are not equal to status_value
'''
data_structure = numpy.dtype(
DATA_STRUCTURE.descr + [
('rx_saturation', 'u4', (self.nchannels,)),
('chan_offset', 'u4', (2 * self.nchannels,)),
('rx_amp', 'u4', (self.nchannels,)),
('rx_snr', 'f4', (self.nchannels,)),
('cross_snr', 'f4', (self.kchan,)),
('sea_power_relative', 'f4', (self.kchan,))]
)
data = numpy.fromfile(self.fp, data_structure, self.nranges)
height = data['range']
José Chávez
formatting
 r1083 winds = numpy.array(
(data['zonal'], data['meridional'], data['vertical']))
Juan C. Espinoza
BLTRParamreader ready
 r1010 snr = data['rx_snr'].T
José Chávez
formatting
 r1083 winds[numpy.where(winds == -9999.)] = numpy.nan
Juan C. Espinoza
BLTRParamreader ready
 r1010 winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
snr[numpy.where(snr == -9999.)] = numpy.nan
snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
José Chávez
formatting
 r1083 snr = numpy.power(10, snr / 10)
Juan C. Espinoza
BLTRParamreader ready
 r1010 return height, winds, snr
def set_output(self):
'''
Storing data from databuffer to dataOut object
'''
José Chávez
formatting
 r1083
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.dataOut.data_SNR = self.snr
Juan C. Espinoza
BLTR ok
 r1018 self.dataOut.height = self.height
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.dataOut.data = self.buffer
Juan C. Espinoza
BLTR ok
 r1018 self.dataOut.utctimeInit = self.time
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.dataOut.utctime = self.dataOut.utctimeInit
self.dataOut.useLocalTime = False
self.dataOut.paramInterval = 157
self.dataOut.timezone = self.timezone
self.dataOut.site = self.siteFile
José Chávez
formatting
 r1083 self.dataOut.nrecords = self.nrecords / self.nmodes
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.dataOut.sizeOfFile = self.sizeOfFile
self.dataOut.lat = self.lat
Juan C. Espinoza
BLTR ok
 r1018 self.dataOut.lon = self.lon
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.dataOut.channelList = range(self.nchannels)
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.dataOut.kchan = self.kchan
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.dataOut.delta = self.delta
self.dataOut.correction = self.correction
self.dataOut.nmodes = self.nmodes
self.dataOut.imode = self.imode
self.dataOut.antenna = self.antenna
self.dataOut.rx_gains = self.rx_gains
self.dataOut.flagNoData = False
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
Juan C. Espinoza
BLTRParamreader ready
 r1010
def getData(self):
'''
Storing data from databuffer to dataOut object
'''
if self.flagNoMoreFiles:
self.dataOut.flagNoData = True
Juan C. Espinoza
Fix utc times and bugs in BLTR modules
 r1085 log.success('No file left to process', 'BLTRParamReader')
Juan C. Espinoza
BLTRParamreader ready
 r1010 return 0
José Chávez
formatting
 r1083 if not self.readNextBlock():
Juan C. Espinoza
BLTRParamreader ready
 r1010 self.dataOut.flagNoData = True
return 0
self.set_output()
return 1