diff --git a/schainpy/model/io/__init__.py b/schainpy/model/io/__init__.py index 2bcee44..4bb334f 100644 --- a/schainpy/model/io/__init__.py +++ b/schainpy/model/io/__init__.py @@ -16,7 +16,7 @@ from .jroIO_hf import * from .jroIO_madrigal import * from .bltrIO_param import * -from .jroIO_bltr import * +from .bltrIO_spectra import * from .jroIO_mira35c import * from .julIO_param import * diff --git a/schainpy/model/io/bltrIO_spectra.py b/schainpy/model/io/bltrIO_spectra.py new file mode 100644 index 0000000..f3b8f4f --- /dev/null +++ b/schainpy/model/io/bltrIO_spectra.py @@ -0,0 +1,466 @@ +import os +import sys +import glob +import fnmatch +import datetime +import time +import re +import h5py +import numpy + +import pylab as plb +from scipy.optimize import curve_fit +from scipy import asarray as ar, exp + +SPEED_OF_LIGHT = 299792458 +SPEED_OF_LIGHT = 3e8 + +try: + from gevent import sleep +except: + from time import sleep + +from .utils import folder_in_range + +from schainpy.model.data.jrodata import Spectra +from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator +from schainpy.utils import log +from schainpy.model.io.jroIO_base import JRODataReader + +def pol2cart(rho, phi): + x = rho * numpy.cos(phi) + y = rho * numpy.sin(phi) + return(x, y) + +FILE_STRUCTURE = numpy.dtype([ # HEADER 48bytes + ('FileMgcNumber', ' vertical) + ('BeamAngleZen', ' endFp: + sys.stderr.write( + "Warning %s: Size value read from System Header is lower than it has to be\n" % fp) + return 0 + + if OffRHeader < endFp: + sys.stderr.write( + "Warning %s: Size value read from System Header size is greater than it has to be\n" % fp) + return 0 + + return 1 + +@MPDecorator +class BLTRSpectraReader (ProcessingUnit): + + def __init__(self): + + ProcessingUnit.__init__(self) + + self.ext = ".fdt" + self.optchar = "P" + self.fpFile = None + self.fp = None + self.BlockCounter = 0 + self.fileSizeByHeader = None + self.filenameList = [] + self.fileSelector = 0 + self.Off2StartNxtRec = 0 + self.RecCounter = 0 + self.flagNoMoreFiles = 0 + self.data_spc = None + self.data_cspc = None + self.path = None + self.OffsetStartHeader = 0 + self.Off2StartData = 0 + self.ipp = 0 + self.nFDTdataRecors = 0 + self.blocksize = 0 + self.dataOut = Spectra() + self.dataOut.flagNoData = False + + def search_files(self): + ''' + Function that indicates the number of .fdt files that exist in the folder to be read. + It also creates an organized list with the names of the files to read. + ''' + + files = glob.glob(os.path.join(self.path, '*{}'.format(self.ext))) + files = sorted(files) + for f in files: + filename = f.split('/')[-1] + if folder_in_range(filename.split('.')[1], self.startDate, self.endDate, '%Y%m%d'): + self.filenameList.append(f) + + def run(self, **kwargs): + ''' + This method will be the one that will initiate the data entry, will be called constantly. + You should first verify that your Setup () is set up and then continue to acquire + the data to be processed with getData (). + ''' + if not self.isConfig: + self.setup(**kwargs) + self.isConfig = True + + self.getData() + + def setup(self, + path=None, + startDate=None, + endDate=None, + startTime=None, + endTime=None, + walk=True, + code=None, + online=False, + mode=None, + **kwargs): + + self.isConfig = True + + self.path = path + self.startDate = startDate + self.endDate = endDate + self.startTime = startTime + self.endTime = endTime + self.walk = walk + self.mode = int(mode) + self.search_files() + if self.filenameList: + self.readFile() + + def getData(self): + ''' + Before starting this function, you should check that there is still an unread file, + If there are still blocks to read or if the data block is empty. + + You should call the file "read". + + ''' + + if self.flagNoMoreFiles: + self.dataOut.flagNoData = True + self.dataOut.error = 'No more files' + + self.readBlock() + + def readFile(self): + ''' + You must indicate if you are reading in Online or Offline mode and load the + The parameters for this file reading mode. + + Then you must do 2 actions: + + 1. Get the BLTR FileHeader. + 2. Start reading the first block. + ''' + + if self.fileSelector < len(self.filenameList): + log.success('Opening file: {}'.format(self.filenameList[self.fileSelector]), self.name) + self.fp = open(self.filenameList[self.fileSelector]) + self.fheader = FileHeaderBLTR(self.fp) + self.rheader = RecordHeaderBLTR(self.fp) + self.nFDTdataRecors = self.fheader.nFDTdataRecors + self.fileSelector += 1 + self.BlockCounter = 0 + return 1 + else: + self.flagNoMoreFiles = True + self.dataOut.flagNoData = True + return 0 + + def readBlock(self): + ''' + It should be checked if the block has data, if it is not passed to the next file. + + Then the following is done: + + 1. Read the RecordHeader + 2. Fill the buffer with the current block number. + + ''' + + if self.BlockCounter == self.nFDTdataRecors: + if not self.readFile(): + return + + if self.mode == 1: + self.rheader.read(self.BlockCounter+1) + elif self.mode == 0: + self.rheader.read(self.BlockCounter) + + self.RecCounter = self.rheader.RecCounter + self.OffsetStartHeader = self.rheader.OffsetStartHeader + self.Off2StartNxtRec = self.rheader.Off2StartNxtRec + self.Off2StartData = self.rheader.Off2StartData + self.nProfiles = self.rheader.nProfiles + self.nChannels = self.rheader.nChannels + self.nHeights = self.rheader.nHeights + self.frequency = self.rheader.TransmitFrec + self.DualModeIndex = self.rheader.DualModeIndex + self.pairsList = [(0, 1), (0, 2), (1, 2)] + self.dataOut.pairsList = self.pairsList + self.nRdPairs = len(self.dataOut.pairsList) + self.dataOut.nRdPairs = self.nRdPairs + self.dataOut.heightList = (self.rheader.StartRangeSamp + numpy.arange(self.nHeights) * self.rheader.SampResolution) / 1000. + self.dataOut.channelList = range(self.nChannels) + self.dataOut.nProfiles=self.rheader.nProfiles + self.dataOut.nIncohInt=self.rheader.nIncohInt + self.dataOut.nCohInt=self.rheader.nCohInt + self.dataOut.ippSeconds= 1/float(self.rheader.PRFhz) + self.dataOut.PRF=self.rheader.PRFhz + self.dataOut.nFFTPoints=self.rheader.nProfiles + self.dataOut.utctime=self.rheader.nUtime + self.dataOut.timeZone = 0 + self.dataOut.useLocalTime = False + self.dataOut.nmodes = 2 + log.log('Reading block {} - {}'.format(self.BlockCounter, self.dataOut.datatime), self.name) + OffDATA = self.OffsetStartHeader + self.RecCounter * \ + self.Off2StartNxtRec + self.Off2StartData + self.fp.seek(OffDATA, os.SEEK_SET) + + self.data_fft = numpy.fromfile(self.fp, [('complex',' vertical) - ('BeamAngleZen', ' endFp: - sys.stderr.write( - "Warning %s: Size value read from System Header is lower than it has to be\n" % fp) - return 0 - - if OffRHeader < endFp: - sys.stderr.write( - "Warning %s: Size value read from System Header size is greater than it has to be\n" % fp) - return 0 - - return 1 - - -class BLTRSpectraReader (ProcessingUnit, FileHeaderBLTR, RecordHeaderBLTR, JRODataReader): - - path = None - startDate = None - endDate = None - startTime = None - endTime = None - walk = None - isConfig = False - - fileList = None - - # metadata - TimeZone = None - Interval = None - heightList = None - - # data - data = None - utctime = None - - def __init__(self, **kwargs): - - # Eliminar de la base la herencia - ProcessingUnit.__init__(self, **kwargs) - - #self.isConfig = False - - #self.pts2read_SelfSpectra = 0 - #self.pts2read_CrossSpectra = 0 - #self.pts2read_DCchannels = 0 - #self.datablock = None - self.utc = None - self.ext = ".fdt" - self.optchar = "P" - self.fpFile = None - self.fp = None - self.BlockCounter = 0 - self.dtype = None - self.fileSizeByHeader = None - self.filenameList = [] - self.fileSelector = 0 - self.Off2StartNxtRec = 0 - self.RecCounter = 0 - self.flagNoMoreFiles = 0 - self.data_spc = None - self.data_cspc = None - self.data_output = None - self.path = None - self.OffsetStartHeader = 0 - self.Off2StartData = 0 - self.ipp = 0 - self.nFDTdataRecors = 0 - self.blocksize = 0 - self.dataOut = Spectra() - self.profileIndex = 1 # Always - self.dataOut.flagNoData = False - self.dataOut.nRdPairs = 0 - self.dataOut.data_spc = None - self.dataOut.velocityX = [] - self.dataOut.velocityY = [] - self.dataOut.velocityV = [] - - def Files2Read(self, fp): - ''' - Function that indicates the number of .fdt files that exist in the folder to be read. - It also creates an organized list with the names of the files to read. - ''' - # self.__checkPath() - - # Gets the list of files within the fp address - ListaData = os.listdir(fp) - # Sort the list of files from least to largest by names - ListaData = sorted(ListaData) - nFiles = 0 # File Counter - FileList = [] # A list is created that will contain the .fdt files - for IndexFile in ListaData: - if '.fdt' in IndexFile: - FileList.append(IndexFile) - nFiles += 1 - - self.filenameList = FileList # List of files from least to largest by names - - def run(self, **kwargs): - ''' - This method will be the one that will initiate the data entry, will be called constantly. - You should first verify that your Setup () is set up and then continue to acquire - the data to be processed with getData (). - ''' - if not self.isConfig: - self.setup(**kwargs) - self.isConfig = True - - self.getData() - - def setup(self, path=None, - startDate=None, - endDate=None, - startTime=None, - endTime=None, - walk=True, - timezone='utc', - code=None, - online=False, - ReadMode=None, - **kwargs): - - self.isConfig = True - - self.path = path - self.startDate = startDate - self.endDate = endDate - self.startTime = startTime - self.endTime = endTime - self.walk = walk - self.ReadMode = int(ReadMode) - - pass - - def getData(self): - ''' - Before starting this function, you should check that there is still an unread file, - If there are still blocks to read or if the data block is empty. - - You should call the file "read". - - ''' - - if self.flagNoMoreFiles: - self.dataOut.flagNoData = True - return 0 - - self.fp = self.path - self.Files2Read(self.fp) - self.readFile(self.fp) - self.dataOut.data_spc = self.data_spc - self.dataOut.data_cspc =self.data_cspc - self.dataOut.data_output=self.data_output - - return self.dataOut.data_spc - - - def readFile(self,fp): - ''' - You must indicate if you are reading in Online or Offline mode and load the - The parameters for this file reading mode. - - Then you must do 2 actions: - - 1. Get the BLTR FileHeader. - 2. Start reading the first block. - ''' - - if self.fileSelector < len(self.filenameList): - - self.fpFile = str(fp) + '/' + \ - str(self.filenameList[self.fileSelector]) - fheader = FileHeaderBLTR() - fheader.FHread(self.fpFile) # Bltr FileHeader Reading - self.nFDTdataRecors = fheader.nFDTdataRecors - - self.readBlock() # Block reading - else: - self.flagNoMoreFiles=True - self.dataOut.flagNoData = True - return 0 - - def getVelRange(self, extrapoints=0): - Lambda = SPEED_OF_LIGHT / 50000000 - # 1./(self.dataOut.ippSeconds * self.dataOut.nCohInt) - PRF = self.dataOut.PRF - Vmax = -Lambda / (4. * (1. / PRF) * self.dataOut.nCohInt * 2.) - deltafreq = PRF / (self.nProfiles) - deltavel = (Vmax * 2) / (self.nProfiles) - freqrange = deltafreq * \ - (numpy.arange(self.nProfiles) - self.nProfiles / 2.) - deltafreq / 2 - velrange = deltavel * \ - (numpy.arange(self.nProfiles) - self.nProfiles / 2.) - return velrange - - def readBlock(self): - ''' - It should be checked if the block has data, if it is not passed to the next file. - - Then the following is done: - - 1. Read the RecordHeader - 2. Fill the buffer with the current block number. - - ''' - - if self.BlockCounter < self.nFDTdataRecors-1: - if self.ReadMode==1: - rheader = RecordHeaderBLTR(RecCounter=self.BlockCounter+1) - elif self.ReadMode==0: - rheader = RecordHeaderBLTR(RecCounter=self.BlockCounter) - - rheader.RHread(self.fpFile) # Bltr FileHeader Reading - - self.OffsetStartHeader = rheader.OffsetStartHeader - self.RecCounter = rheader.RecCounter - self.Off2StartNxtRec = rheader.Off2StartNxtRec - self.Off2StartData = rheader.Off2StartData - self.nProfiles = rheader.nProfiles - self.nChannels = rheader.nChannels - self.nHeights = rheader.nHeights - self.frequency = rheader.TransmitFrec - self.DualModeIndex = rheader.DualModeIndex - - self.pairsList = [(0, 1), (0, 2), (1, 2)] - self.dataOut.pairsList = self.pairsList - - self.nRdPairs = len(self.dataOut.pairsList) - self.dataOut.nRdPairs = self.nRdPairs - self.__firstHeigth=rheader.StartRangeSamp - self.__deltaHeigth=rheader.SampResolution - self.dataOut.heightList= self.__firstHeigth + numpy.array(range(self.nHeights))*self.__deltaHeigth - self.dataOut.channelList = range(self.nChannels) - self.dataOut.nProfiles=rheader.nProfiles - self.dataOut.nIncohInt=rheader.nIncohInt - self.dataOut.nCohInt=rheader.nCohInt - self.dataOut.ippSeconds= 1/float(rheader.PRFhz) - self.dataOut.PRF=rheader.PRFhz - self.dataOut.nFFTPoints=rheader.nProfiles - self.dataOut.utctime=rheader.nUtime - self.dataOut.timeZone=0 - self.dataOut.normFactor= self.dataOut.nProfiles*self.dataOut.nIncohInt*self.dataOut.nCohInt - self.dataOut.outputInterval= self.dataOut.ippSeconds * self.dataOut.nCohInt * self.dataOut.nIncohInt * self.nProfiles - - self.data_output=numpy.ones([3,rheader.nHeights])*numpy.NaN - self.dataOut.velocityX=[] - self.dataOut.velocityY=[] - self.dataOut.velocityV=[] - - '''Block Reading, the Block Data is received and Reshape is used to give it - shape. - ''' - - # Procedure to take the pointer to where the date block starts - startDATA = open(self.fpFile, "rb") - OffDATA = self.OffsetStartHeader + self.RecCounter * \ - self.Off2StartNxtRec + self.Off2StartData - startDATA.seek(OffDATA, os.SEEK_SET) - - def moving_average(x, N=2): - return numpy.convolve(x, numpy.ones((N,)) / N)[(N - 1):] - - def gaus(xSamples, a, x0, sigma): - return a * exp(-(xSamples - x0)**2 / (2 * sigma**2)) - - def Find(x, value): - for index in range(len(x)): - if x[index] == value: - return index - - def pol2cart(rho, phi): - x = rho * numpy.cos(phi) - y = rho * numpy.sin(phi) - return(x, y) - - if self.DualModeIndex==self.ReadMode: - - self.data_fft = numpy.fromfile( startDATA, [('complex','