import os import sys import glob import fnmatch import datetime import time import re import h5py import numpy import pylab as plb from scipy.optimize import curve_fit from scipy import asarray as ar, exp from scipy import stats from numpy.ma.core import getdata SPEED_OF_LIGHT = 299792458 SPEED_OF_LIGHT = 3e8 try: from gevent import sleep except: from time import sleep from schainpy.model.data.jrodata import Spectra #from schainpy.model.data.BLTRheaderIO import FileHeader, RecordHeader from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation #from schainpy.model.io.jroIO_bltr import BLTRReader from numpy import imag, shape, NaN from .jroIO_base import JRODataReader class Header(object): def __init__(self): raise NotImplementedError def read(self): raise NotImplementedError def write(self): raise NotImplementedError def printInfo(self): message = "#" * 50 + "\n" message += self.__class__.__name__.upper() + "\n" message += "#" * 50 + "\n" keyList = list(self.__dict__.keys()) keyList.sort() for key in keyList: message += "%s = %s" % (key, self.__dict__[key]) + "\n" if "size" not in keyList: attr = getattr(self, "size") if attr: message += "%s = %s" % ("size", attr) + "\n" FILE_STRUCTURE = numpy.dtype([ # HEADER 48bytes ('FileMgcNumber', ' vertical) ('BeamAngleZen', ' endFp: sys.stderr.write( "Warning %s: Size value read from System Header is lower than it has to be\n" % fp) return 0 if OffRHeader < endFp: sys.stderr.write( "Warning %s: Size value read from System Header size is greater than it has to be\n" % fp) return 0 return 1 class BLTRSpectraReader (ProcessingUnit, FileHeaderBLTR, RecordHeaderBLTR, JRODataReader): path = None startDate = None endDate = None startTime = None endTime = None walk = None isConfig = False fileList = None # metadata TimeZone = None Interval = None heightList = None # data data = None utctime = None def __init__(self, **kwargs): # Eliminar de la base la herencia ProcessingUnit.__init__(self, **kwargs) #self.isConfig = False #self.pts2read_SelfSpectra = 0 #self.pts2read_CrossSpectra = 0 #self.pts2read_DCchannels = 0 #self.datablock = None self.utc = None self.ext = ".fdt" self.optchar = "P" self.fpFile = None self.fp = None self.BlockCounter = 0 self.dtype = None self.fileSizeByHeader = None self.filenameList = [] self.fileSelector = 0 self.Off2StartNxtRec = 0 self.RecCounter = 0 self.flagNoMoreFiles = 0 self.data_spc = None self.data_cspc = None self.data_output = None self.path = None self.OffsetStartHeader = 0 self.Off2StartData = 0 self.ipp = 0 self.nFDTdataRecors = 0 self.blocksize = 0 self.dataOut = Spectra() self.profileIndex = 1 # Always self.dataOut.flagNoData = False self.dataOut.nRdPairs = 0 self.dataOut.data_spc = None self.dataOut.velocityX = [] self.dataOut.velocityY = [] self.dataOut.velocityV = [] def Files2Read(self, fp): ''' Function that indicates the number of .fdt files that exist in the folder to be read. It also creates an organized list with the names of the files to read. ''' # self.__checkPath() # Gets the list of files within the fp address ListaData = os.listdir(fp) # Sort the list of files from least to largest by names ListaData = sorted(ListaData) nFiles = 0 # File Counter FileList = [] # A list is created that will contain the .fdt files for IndexFile in ListaData: if '.fdt' in IndexFile: FileList.append(IndexFile) nFiles += 1 self.filenameList = FileList # List of files from least to largest by names def run(self, **kwargs): ''' This method will be the one that will initiate the data entry, will be called constantly. You should first verify that your Setup () is set up and then continue to acquire the data to be processed with getData (). ''' if not self.isConfig: self.setup(**kwargs) self.isConfig = True self.getData() def setup(self, path=None, startDate=None, endDate=None, startTime=None, endTime=None, walk=True, timezone='utc', code=None, online=False, ReadMode=None, **kwargs): self.isConfig = True self.path = path self.startDate = startDate self.endDate = endDate self.startTime = startTime self.endTime = endTime self.walk = walk self.ReadMode = int(ReadMode) pass def getData(self): ''' Before starting this function, you should check that there is still an unread file, If there are still blocks to read or if the data block is empty. You should call the file "read". ''' if self.flagNoMoreFiles: self.dataOut.flagNoData = True return 0 self.fp = self.path self.Files2Read(self.fp) self.readFile(self.fp) self.dataOut.data_spc = self.data_spc self.dataOut.data_cspc =self.data_cspc self.dataOut.data_output=self.data_output return self.dataOut.data_spc def readFile(self,fp): ''' You must indicate if you are reading in Online or Offline mode and load the The parameters for this file reading mode. Then you must do 2 actions: 1. Get the BLTR FileHeader. 2. Start reading the first block. ''' if self.fileSelector < len(self.filenameList): self.fpFile = str(fp) + '/' + \ str(self.filenameList[self.fileSelector]) fheader = FileHeaderBLTR() fheader.FHread(self.fpFile) # Bltr FileHeader Reading self.nFDTdataRecors = fheader.nFDTdataRecors self.readBlock() # Block reading else: self.flagNoMoreFiles=True self.dataOut.flagNoData = True return 0 def getVelRange(self, extrapoints=0): Lambda = SPEED_OF_LIGHT / 50000000 # 1./(self.dataOut.ippSeconds * self.dataOut.nCohInt) PRF = self.dataOut.PRF Vmax = -Lambda / (4. * (1. / PRF) * self.dataOut.nCohInt * 2.) deltafreq = PRF / (self.nProfiles) deltavel = (Vmax * 2) / (self.nProfiles) freqrange = deltafreq * \ (numpy.arange(self.nProfiles) - self.nProfiles / 2.) - deltafreq / 2 velrange = deltavel * \ (numpy.arange(self.nProfiles) - self.nProfiles / 2.) return velrange def readBlock(self): ''' It should be checked if the block has data, if it is not passed to the next file. Then the following is done: 1. Read the RecordHeader 2. Fill the buffer with the current block number. ''' if self.BlockCounter < self.nFDTdataRecors-1: if self.ReadMode==1: rheader = RecordHeaderBLTR(RecCounter=self.BlockCounter+1) elif self.ReadMode==0: rheader = RecordHeaderBLTR(RecCounter=self.BlockCounter) rheader.RHread(self.fpFile) # Bltr FileHeader Reading self.OffsetStartHeader = rheader.OffsetStartHeader self.RecCounter = rheader.RecCounter self.Off2StartNxtRec = rheader.Off2StartNxtRec self.Off2StartData = rheader.Off2StartData self.nProfiles = rheader.nProfiles self.nChannels = rheader.nChannels self.nHeights = rheader.nHeights self.frequency = rheader.TransmitFrec self.DualModeIndex = rheader.DualModeIndex self.pairsList = [(0, 1), (0, 2), (1, 2)] self.dataOut.pairsList = self.pairsList self.nRdPairs = len(self.dataOut.pairsList) self.dataOut.nRdPairs = self.nRdPairs self.__firstHeigth=rheader.StartRangeSamp self.__deltaHeigth=rheader.SampResolution self.dataOut.heightList= self.__firstHeigth + numpy.array(range(self.nHeights))*self.__deltaHeigth self.dataOut.channelList = range(self.nChannels) self.dataOut.nProfiles=rheader.nProfiles self.dataOut.nIncohInt=rheader.nIncohInt self.dataOut.nCohInt=rheader.nCohInt self.dataOut.ippSeconds= 1/float(rheader.PRFhz) self.dataOut.PRF=rheader.PRFhz self.dataOut.nFFTPoints=rheader.nProfiles self.dataOut.utctime=rheader.nUtime self.dataOut.timeZone=0 self.dataOut.normFactor= self.dataOut.nProfiles*self.dataOut.nIncohInt*self.dataOut.nCohInt self.dataOut.outputInterval= self.dataOut.ippSeconds * self.dataOut.nCohInt * self.dataOut.nIncohInt * self.nProfiles self.data_output=numpy.ones([3,rheader.nHeights])*numpy.NaN self.dataOut.velocityX=[] self.dataOut.velocityY=[] self.dataOut.velocityV=[] '''Block Reading, the Block Data is received and Reshape is used to give it shape. ''' # Procedure to take the pointer to where the date block starts startDATA = open(self.fpFile, "rb") OffDATA = self.OffsetStartHeader + self.RecCounter * \ self.Off2StartNxtRec + self.Off2StartData startDATA.seek(OffDATA, os.SEEK_SET) def moving_average(x, N=2): return numpy.convolve(x, numpy.ones((N,)) / N)[(N - 1):] def gaus(xSamples, a, x0, sigma): return a * exp(-(xSamples - x0)**2 / (2 * sigma**2)) def Find(x, value): for index in range(len(x)): if x[index] == value: return index def pol2cart(rho, phi): x = rho * numpy.cos(phi) y = rho * numpy.sin(phi) return(x, y) if self.DualModeIndex==self.ReadMode: self.data_fft = numpy.fromfile( startDATA, [('complex','