bltrIO_param.py
361 lines
| 11.2 KiB
| text/x-python
|
PythonLexer
Juan C. Espinoza
|
r1010 | ''' | ||
| Created on Nov 9, 2016 | ||||
| @author: roj- LouVD | ||||
| ''' | ||||
| import os | ||||
| import sys | ||||
| import time | ||||
| import glob | ||||
| import datetime | ||||
|
Juan C. Espinoza
|
r1018 | |||
|
Juan C. Espinoza
|
r1010 | import numpy | ||
| r1241 | import schainpy.admin | |||
|
Juan C. Espinoza
|
r1181 | from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator | ||
|
Juan C. Espinoza
|
r1010 | from schainpy.model.data.jrodata import Parameters | ||
|
Juan C. Espinoza
|
r1322 | from schainpy.model.io.jroIO_base import Reader | ||
|
Juan C. Espinoza
|
r1085 | from schainpy.utils import log | ||
|
Juan C. Espinoza
|
r1010 | |||
| FILE_HEADER_STRUCTURE = numpy.dtype([ | ||||
| ('FMN', '<u4'), | ||||
| ('nrec', '<u4'), | ||||
| ('fr_offset', '<u4'), | ||||
| ('id', '<u4'), | ||||
| ('site', 'u1', (32,)) | ||||
| ]) | ||||
| REC_HEADER_STRUCTURE = numpy.dtype([ | ||||
| ('rmn', '<u4'), | ||||
| ('rcounter', '<u4'), | ||||
| ('nr_offset', '<u4'), | ||||
| ('tr_offset', '<u4'), | ||||
| ('time', '<u4'), | ||||
| ('time_msec', '<u4'), | ||||
| ('tag', 'u1', (32,)), | ||||
| ('comments', 'u1', (32,)), | ||||
| ('lat', '<f4'), | ||||
| ('lon', '<f4'), | ||||
| ('gps_status', '<u4'), | ||||
| ('freq', '<u4'), | ||||
| ('freq0', '<u4'), | ||||
| ('nchan', '<u4'), | ||||
| ('delta_r', '<u4'), | ||||
| ('nranges', '<u4'), | ||||
| ('r0', '<u4'), | ||||
| ('prf', '<u4'), | ||||
| ('ncoh', '<u4'), | ||||
| ('npoints', '<u4'), | ||||
| ('polarization', '<i4'), | ||||
| ('rx_filter', '<u4'), | ||||
| ('nmodes', '<u4'), | ||||
| ('dmode_index', '<u4'), | ||||
| ('dmode_rngcorr', '<u4'), | ||||
| ('nrxs', '<u4'), | ||||
| ('acf_length', '<u4'), | ||||
| ('acf_lags', '<u4'), | ||||
| ('sea_to_atmos', '<f4'), | ||||
| ('sea_notch', '<u4'), | ||||
| ('lh_sea', '<u4'), | ||||
| ('hh_sea', '<u4'), | ||||
| ('nbins_sea', '<u4'), | ||||
| ('min_snr', '<f4'), | ||||
| ('min_cc', '<f4'), | ||||
| ('max_time_diff', '<f4') | ||||
| ]) | ||||
| DATA_STRUCTURE = numpy.dtype([ | ||||
| ('range', '<u4'), | ||||
| ('status', '<u4'), | ||||
| ('zonal', '<f4'), | ||||
| ('meridional', '<f4'), | ||||
| ('vertical', '<f4'), | ||||
| ('zonal_a', '<f4'), | ||||
| ('meridional_a', '<f4'), | ||||
| ('corrected_fading', '<f4'), # seconds | ||||
| ('uncorrected_fading', '<f4'), # seconds | ||||
| ('time_diff', '<f4'), | ||||
| ('major_axis', '<f4'), | ||||
| ('axial_ratio', '<f4'), | ||||
| ('orientation', '<f4'), | ||||
| ('sea_power', '<u4'), | ||||
| ('sea_algorithm', '<u4') | ||||
| ]) | ||||
|
Juan C. Espinoza
|
r1287 | |||
|
Juan C. Espinoza
|
r1322 | class BLTRParamReader(Reader, ProcessingUnit): | ||
|
Juan C. Espinoza
|
r1010 | ''' | ||
| r1737 | Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR | |||
|
Juan C. Espinoza
|
r1217 | from *.sswma files | ||
|
Juan C. Espinoza
|
r1010 | ''' | ||
| ext = '.sswma' | ||||
George Yong
|
r1185 | def __init__(self): | ||
|
Juan C. Espinoza
|
r1010 | |||
|
George Yong
|
r1185 | ProcessingUnit.__init__(self) | ||
|
Juan C. Espinoza
|
r1010 | |||
José Chávez
|
r1083 | self.dataOut = Parameters() | ||
|
Juan C. Espinoza
|
r1322 | self.dataOut.timezone = 300 | ||
|
Juan C. Espinoza
|
r1010 | self.counter_records = 0 | ||
| self.flagNoMoreFiles = 0 | ||||
|
José Chávez
|
r1083 | self.isConfig = False | ||
|
Juan C. Espinoza
|
r1010 | self.filename = None | ||
|
Juan C. Espinoza
|
r1322 | self.status_value = 0 | ||
|
Juan C. Espinoza
|
r1085 | self.datatime = datetime.datetime(1900,1,1) | ||
|
Juan C. Espinoza
|
r1322 | self.filefmt = "*********%Y%m%d******" | ||
| def setup(self, **kwargs): | ||||
| r1737 | ||||
|
Juan C. Espinoza
|
r1322 | self.set_kwargs(**kwargs) | ||
| r1737 | ||||
|
Juan C. Espinoza
|
r1010 | if self.path is None: | ||
|
George Yong
|
r1167 | raise ValueError("The path is not valid") | ||
|
Juan C. Espinoza
|
r1010 | |||
|
Juan C. Espinoza
|
r1322 | if self.online: | ||
| log.log("Searching files in online mode...", self.name) | ||||
| for nTries in range(self.nTries): | ||||
| fullpath = self.searchFilesOnLine(self.path, self.startDate, | ||||
| r1737 | self.endDate, self.expLabel, self.ext, self.walk, | |||
|
Juan C. Espinoza
|
r1322 | self.filefmt, self.folderfmt) | ||
| try: | ||||
| fullpath = next(fullpath) | ||||
| except: | ||||
| fullpath = None | ||||
| r1737 | ||||
|
Juan C. Espinoza
|
r1322 | if fullpath: | ||
| self.fileSize = os.path.getsize(fullpath) | ||||
| self.filename = fullpath | ||||
| self.flagIsNewFile = 1 | ||||
| if self.fp != None: | ||||
| self.fp.close() | ||||
| self.fp = self.open_file(fullpath, self.open_mode) | ||||
| self.flagNoMoreFiles = 0 | ||||
| break | ||||
|
Juan C. Espinoza
|
r1010 | |||
|
Juan C. Espinoza
|
r1322 | log.warning( | ||
| 'Waiting {} sec for a valid file in {}: try {} ...'.format( | ||||
| r1737 | self.delay, self.path, nTries + 1), | |||
|
Juan C. Espinoza
|
r1322 | self.name) | ||
| time.sleep(self.delay) | ||||
|
Juan C. Espinoza
|
r1010 | |||
|
Juan C. Espinoza
|
r1322 | if not(fullpath): | ||
| raise schainpy.admin.SchainError( | ||||
| r1737 | 'There isn\'t any valid file in {}'.format(self.path)) | |||
|
Juan C. Espinoza
|
r1322 | self.readFirstHeader() | ||
| else: | ||||
| log.log("Searching files in {}".format(self.path), self.name) | ||||
| r1737 | self.filenameList = self.searchFilesOffLine(self.path, self.startDate, | |||
|
Juan C. Espinoza
|
r1322 | self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt) | ||
| self.setNextFile() | ||||
|
José Chávez
|
r1083 | |||
|
Juan C. Espinoza
|
r1322 | def checkForRealPath(self, nextFile, nextDay): | ||
|
Juan C. Espinoza
|
r1215 | ''' | ||
| ''' | ||||
|
Juan C. Espinoza
|
r1322 | dt = self.datatime + datetime.timedelta(1) | ||
| filename = '{}.{}{}'.format(self.siteFile, dt.strftime('%Y%m%d'), self.ext) | ||||
| fullfilename = os.path.join(self.path, filename) | ||||
| if os.path.exists(fullfilename): | ||||
| return fullfilename, filename | ||||
| return None, filename | ||||
| r1737 | ||||
|
Juan C. Espinoza
|
r1322 | def readFirstHeader(self): | ||
|
Juan C. Espinoza
|
r1010 | ''' | ||
|
José Chávez
|
r1083 | ''' | ||
| # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya | ||||
|
Juan C. Espinoza
|
r1322 | self.siteFile = self.filename.split('/')[-1].split('.')[0] | ||
|
Juan C. Espinoza
|
r1010 | self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1) | ||
| self.nrecords = self.header_file['nrec'][0] | ||||
| self.counter_records = 0 | ||||
| self.flagIsNewFile = 0 | ||||
| r1737 | self.fileIndex += 1 | |||
|
Juan C. Espinoza
|
r1010 | |||
| def readNextBlock(self): | ||||
|
Juan C. Espinoza
|
r1085 | while True: | ||
|
Juan C. Espinoza
|
r1217 | if not self.online and self.counter_records == self.nrecords: | ||
|
Juan C. Espinoza
|
r1010 | self.flagIsNewFile = 1 | ||
| if not self.setNextFile(): | ||||
| return 0 | ||||
|
Juan C. Espinoza
|
r1215 | try: | ||
| r1737 | if self.online and self.counter_records == 0: | |||
| pos = int(self.fileSize / (38512)) | ||||
| self.counter_records = pos*2 - 2 | ||||
| pointer = 38512 * (pos-1) + 48 | ||||
| self.fp.seek(pointer) | ||||
| else: | ||||
| pointer = self.fp.tell() | ||||
|
Juan C. Espinoza
|
r1215 | self.readBlock() | ||
| except: | ||||
|
Juan C. Espinoza
|
r1219 | if self.online and self.waitDataBlock(pointer, 38512) == 1: | ||
|
Juan C. Espinoza
|
r1215 | continue | ||
| else: | ||||
| if not self.setNextFile(): | ||||
| return 0 | ||||
|
Juan C. Espinoza
|
r1085 | |||
| if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \ | ||||
| (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)): | ||||
| log.warning( | ||||
| 'Reading Record No. {}/{} -> {} [Skipping]'.format( | ||||
| self.counter_records, | ||||
| self.nrecords, | ||||
| self.datatime.ctime()), | ||||
| 'BLTRParamReader') | ||||
|
Juan C. Espinoza
|
r1010 | continue | ||
| break | ||||
|
Juan C. Espinoza
|
r1217 | log.log('Reading Record No. {} -> {}'.format( | ||
|
Juan C. Espinoza
|
r1010 | self.counter_records, | ||
|
Juan C. Espinoza
|
r1085 | self.datatime.ctime()), 'BLTRParamReader') | ||
|
Juan C. Espinoza
|
r1010 | |||
| return 1 | ||||
| def readBlock(self): | ||||
| pointer = self.fp.tell() | ||||
| header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1) | ||||
|
Juan C. Espinoza
|
r1181 | self.nchannels = int(header_rec['nchan'][0] / 2) | ||
|
Juan C. Espinoza
|
r1010 | self.kchan = header_rec['nrxs'][0] | ||
| self.nmodes = header_rec['nmodes'][0] | ||||
| self.nranges = header_rec['nranges'][0] | ||||
| self.fp.seek(pointer) | ||||
| self.height = numpy.empty((self.nmodes, self.nranges)) | ||||
|
George Yong
|
r1185 | self.snr = numpy.empty((self.nmodes, int(self.nchannels), self.nranges)) | ||
|
Juan C. Espinoza
|
r1010 | self.buffer = numpy.empty((self.nmodes, 3, self.nranges)) | ||
|
Juan C. Espinoza
|
r1085 | self.flagDiscontinuousBlock = 0 | ||
|
Juan C. Espinoza
|
r1010 | |||
| for mode in range(self.nmodes): | ||||
|
José Chávez
|
r1083 | self.readHeader() | ||
|
Juan C. Espinoza
|
r1010 | data = self.readData() | ||
| self.height[mode] = (data[0] - self.correction) / 1000. | ||||
| self.buffer[mode] = data[1] | ||||
| self.snr[mode] = data[2] | ||||
| self.counter_records = self.counter_records + self.nmodes | ||||
| return | ||||
| def readHeader(self): | ||||
| ''' | ||||
| RecordHeader of BLTR rawdata file | ||||
| ''' | ||||
|
José Chávez
|
r1083 | |||
|
Juan C. Espinoza
|
r1010 | header_structure = numpy.dtype( | ||
| REC_HEADER_STRUCTURE.descr + [ | ||||
|
George Yong
|
r1185 | ('antenna_coord', 'f4', (2, int(self.nchannels))), | ||
| ('rx_gains', 'u4', (int(self.nchannels),)), | ||||
| ('rx_analysis', 'u4', (int(self.nchannels),)) | ||||
|
Juan C. Espinoza
|
r1010 | ] | ||
| ) | ||||
| self.header_rec = numpy.fromfile(self.fp, header_structure, 1) | ||||
| self.lat = self.header_rec['lat'][0] | ||||
| self.lon = self.header_rec['lon'][0] | ||||
| self.delta = self.header_rec['delta_r'][0] | ||||
| self.correction = self.header_rec['dmode_rngcorr'][0] | ||||
|
José Chávez
|
r1083 | self.imode = self.header_rec['dmode_index'][0] | ||
|
Juan C. Espinoza
|
r1010 | self.antenna = self.header_rec['antenna_coord'] | ||
| r1737 | self.rx_gains = self.header_rec['rx_gains'] | |||
| self.time = self.header_rec['time'][0] | ||||
|
Juan C. Espinoza
|
r1085 | dt = datetime.datetime.utcfromtimestamp(self.time) | ||
| if dt.date()>self.datatime.date(): | ||||
| self.flagDiscontinuousBlock = 1 | ||||
| self.datatime = dt | ||||
| r1737 | ||||
|
Juan C. Espinoza
|
r1010 | def readData(self): | ||
| ''' | ||||
| r1737 | Reading and filtering data block record of BLTR rawdata file, | |||
|
Juan C. Espinoza
|
r1217 | filtering is according to status_value. | ||
|
Juan C. Espinoza
|
r1010 | |||
| Input: | ||||
| r1737 | status_value - Array data is set to NAN for values that are not | |||
|
Juan C. Espinoza
|
r1217 | equal to status_value | ||
|
Juan C. Espinoza
|
r1010 | |||
| ''' | ||||
|
George Yong
|
r1185 | self.nchannels = int(self.nchannels) | ||
|
Juan C. Espinoza
|
r1010 | |||
| data_structure = numpy.dtype( | ||||
| DATA_STRUCTURE.descr + [ | ||||
| ('rx_saturation', 'u4', (self.nchannels,)), | ||||
| ('chan_offset', 'u4', (2 * self.nchannels,)), | ||||
| ('rx_amp', 'u4', (self.nchannels,)), | ||||
| ('rx_snr', 'f4', (self.nchannels,)), | ||||
| ('cross_snr', 'f4', (self.kchan,)), | ||||
| ('sea_power_relative', 'f4', (self.kchan,))] | ||||
| ) | ||||
| data = numpy.fromfile(self.fp, data_structure, self.nranges) | ||||
| height = data['range'] | ||||
|
José Chávez
|
r1083 | winds = numpy.array( | ||
| (data['zonal'], data['meridional'], data['vertical'])) | ||||
|
Juan C. Espinoza
|
r1010 | snr = data['rx_snr'].T | ||
|
José Chávez
|
r1083 | winds[numpy.where(winds == -9999.)] = numpy.nan | ||
|
Juan C. Espinoza
|
r1010 | winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan | ||
| snr[numpy.where(snr == -9999.)] = numpy.nan | ||||
| snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan | ||||
|
José Chávez
|
r1083 | snr = numpy.power(10, snr / 10) | ||
|
Juan C. Espinoza
|
r1010 | return height, winds, snr | ||
| def set_output(self): | ||||
| ''' | ||||
| Storing data from databuffer to dataOut object | ||||
| ''' | ||||
|
José Chávez
|
r1083 | |||
| r1343 | self.dataOut.data_snr = self.snr | |||
|
Juan C. Espinoza
|
r1018 | self.dataOut.height = self.height | ||
|
Juan C. Espinoza
|
r1085 | self.dataOut.data = self.buffer | ||
|
Juan C. Espinoza
|
r1018 | self.dataOut.utctimeInit = self.time | ||
|
Juan C. Espinoza
|
r1010 | self.dataOut.utctime = self.dataOut.utctimeInit | ||
| self.dataOut.useLocalTime = False | ||||
| self.dataOut.paramInterval = 157 | ||||
| self.dataOut.site = self.siteFile | ||||
|
José Chávez
|
r1083 | self.dataOut.nrecords = self.nrecords / self.nmodes | ||
|
Juan C. Espinoza
|
r1010 | self.dataOut.lat = self.lat | ||
|
Juan C. Espinoza
|
r1018 | self.dataOut.lon = self.lon | ||
|
George Yong
|
r1167 | self.dataOut.channelList = list(range(self.nchannels)) | ||
| r1737 | self.dataOut.kchan = self.kchan | |||
|
Juan C. Espinoza
|
r1010 | self.dataOut.delta = self.delta | ||
| self.dataOut.correction = self.correction | ||||
| self.dataOut.nmodes = self.nmodes | ||||
| self.dataOut.imode = self.imode | ||||
| self.dataOut.antenna = self.antenna | ||||
| self.dataOut.rx_gains = self.rx_gains | ||||
| self.dataOut.flagNoData = False | ||||
|
Juan C. Espinoza
|
r1085 | self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock | ||
|
Juan C. Espinoza
|
r1010 | |||
| def getData(self): | ||||
| ''' | ||||
| Storing data from databuffer to dataOut object | ||||
| ''' | ||||
| if self.flagNoMoreFiles: | ||||
| self.dataOut.flagNoData = True | ||||
|
Juan C. Espinoza
|
r1322 | return 0 | ||
|
Juan C. Espinoza
|
r1010 | |||
|
José Chávez
|
r1083 | if not self.readNextBlock(): | ||
|
Juan C. Espinoza
|
r1010 | self.dataOut.flagNoData = True | ||
|
Juan C. Espinoza
|
r1322 | return 0 | ||
|
Juan C. Espinoza
|
r1010 | |||
| self.set_output() | ||||
|
George Yong
|
r1185 | return 1 | ||
| r1737 | ||||
|
Juan C. Espinoza
|
r1322 | def run(self, **kwargs): | ||
| ''' | ||||
| ''' | ||||
| if not(self.isConfig): | ||||
| self.setup(**kwargs) | ||||
| self.isConfig = True | ||||
| self.getData() | ||||
| r1737 | return | |||
