import os, sys import glob import fnmatch import datetime import time import re import h5py import numpy import matplotlib.pyplot as plt import pylab as plb from scipy.optimize import curve_fit from scipy import asarray as ar, exp from scipy import stats from numpy.ma.core import getdata SPEED_OF_LIGHT = 299792458 SPEED_OF_LIGHT = 3e8 try: from gevent import sleep except: from time import sleep from schainpy.model.data.jrodata import Spectra #from schainpy.model.data.BLTRheaderIO import FileHeader, RecordHeader from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation #from schainpy.model.io.jroIO_bltr import BLTRReader from numpy import imag, shape, NaN from jroIO_base import JRODataReader class Header(object): def __init__(self): raise NotImplementedError def read(self): raise NotImplementedError def write(self): raise NotImplementedError def printInfo(self): message = "#"*50 + "\n" message += self.__class__.__name__.upper() + "\n" message += "#"*50 + "\n" keyList = self.__dict__.keys() keyList.sort() for key in keyList: message += "%s = %s" %(key, self.__dict__[key]) + "\n" if "size" not in keyList: attr = getattr(self, "size") if attr: message += "%s = %s" %("size", attr) + "\n" #print message FILE_STRUCTURE = numpy.dtype([ #HEADER 48bytes ('FileMgcNumber',' vertical) ('AntennaCoord0',' endFp: sys.stderr.write("Warning %s: Size value read from System Header is lower than it has to be\n" %fp) return 0 if OffRHeader < endFp: sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" %fp) return 0 return 1 class BLTRSpectraReader (ProcessingUnit, FileHeaderBLTR, RecordHeaderBLTR, JRODataReader): path = None startDate = None endDate = None startTime = None endTime = None walk = None isConfig = False fileList= None #metadata TimeZone= None Interval= None heightList= None #data data= None utctime= None def __init__(self, **kwargs): #Eliminar de la base la herencia ProcessingUnit.__init__(self, **kwargs) #self.isConfig = False #self.pts2read_SelfSpectra = 0 #self.pts2read_CrossSpectra = 0 #self.pts2read_DCchannels = 0 #self.datablock = None self.utc = None self.ext = ".fdt" self.optchar = "P" self.fpFile=None self.fp = None self.BlockCounter=0 self.dtype = None self.fileSizeByHeader = None self.filenameList = [] self.fileSelector = 0 self.Off2StartNxtRec=0 self.RecCounter=0 self.flagNoMoreFiles = 0 self.data_spc=None self.data_cspc=None self.data_output=None self.path = None self.OffsetStartHeader=0 self.Off2StartData=0 self.ipp = 0 self.nFDTdataRecors=0 self.blocksize = 0 self.dataOut = Spectra() self.profileIndex = 1 #Always self.dataOut.flagNoData=False self.dataOut.nRdPairs = 0 self.dataOut.pairsList = [] self.dataOut.data_spc=None self.dataOut.noise=[] self.dataOut.velocityX=[] self.dataOut.velocityY=[] self.dataOut.velocityV=[] def Files2Read(self, fp): ''' Function that indicates the number of .fdt files that exist in the folder to be read. It also creates an organized list with the names of the files to read. ''' #self.__checkPath() ListaData=os.listdir(fp) #Gets the list of files within the fp address ListaData=sorted(ListaData) #Sort the list of files from least to largest by names nFiles=0 #File Counter FileList=[] #A list is created that will contain the .fdt files for IndexFile in ListaData : if '.fdt' in IndexFile: FileList.append(IndexFile) nFiles+=1 #print 'Files2Read' #print 'Existen '+str(nFiles)+' archivos .fdt' self.filenameList=FileList #List of files from least to largest by names def run(self, **kwargs): ''' This method will be the one that will initiate the data entry, will be called constantly. You should first verify that your Setup () is set up and then continue to acquire the data to be processed with getData (). ''' if not self.isConfig: self.setup(**kwargs) self.isConfig = True self.getData() #print 'running' def setup(self, path=None, startDate=None, endDate=None, startTime=None, endTime=None, walk=True, timezone='utc', code = None, online=False, ReadMode=None, **kwargs): self.isConfig = True self.path=path self.startDate=startDate self.endDate=endDate self.startTime=startTime self.endTime=endTime self.walk=walk self.ReadMode=int(ReadMode) pass def getData(self): ''' Before starting this function, you should check that there is still an unread file, If there are still blocks to read or if the data block is empty. You should call the file "read". ''' if self.flagNoMoreFiles: self.dataOut.flagNoData = True print 'NoData se vuelve true' return 0 self.fp=self.path self.Files2Read(self.fp) self.readFile(self.fp) self.dataOut.data_spc = self.data_spc self.dataOut.data_cspc =self.data_cspc self.dataOut.data_output=self.data_output print 'self.dataOut.data_output', shape(self.dataOut.data_output) #self.removeDC() return self.dataOut.data_spc def readFile(self,fp): ''' You must indicate if you are reading in Online or Offline mode and load the The parameters for this file reading mode. Then you must do 2 actions: 1. Get the BLTR FileHeader. 2. Start reading the first block. ''' #The address of the folder is generated the name of the .fdt file that will be read print "File: ",self.fileSelector+1 if self.fileSelector < len(self.filenameList): self.fpFile=str(fp)+'/'+str(self.filenameList[self.fileSelector]) #print self.fpFile fheader = FileHeaderBLTR() fheader.FHread(self.fpFile) #Bltr FileHeader Reading self.nFDTdataRecors=fheader.nFDTdataRecors self.readBlock() #Block reading else: print 'readFile FlagNoData becomes true' self.flagNoMoreFiles=True self.dataOut.flagNoData = True return 0 def getVelRange(self, extrapoints=0): Lambda= SPEED_OF_LIGHT/50000000 PRF = self.dataOut.PRF#1./(self.dataOut.ippSeconds * self.dataOut.nCohInt) Vmax=-Lambda/(4.*(1./PRF)*self.dataOut.nCohInt*2.) deltafreq = PRF / (self.nProfiles) deltavel = (Vmax*2) / (self.nProfiles) freqrange = deltafreq*(numpy.arange(self.nProfiles)-self.nProfiles/2.) - deltafreq/2 velrange = deltavel*(numpy.arange(self.nProfiles)-self.nProfiles/2.) return velrange def readBlock(self): ''' It should be checked if the block has data, if it is not passed to the next file. Then the following is done: 1. Read the RecordHeader 2. Fill the buffer with the current block number. ''' if self.BlockCounter < self.nFDTdataRecors-2: print self.nFDTdataRecors, 'CONDICION!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' if self.ReadMode==1: rheader = RecordHeaderBLTR(RecCounter=self.BlockCounter+1) elif self.ReadMode==0: rheader = RecordHeaderBLTR(RecCounter=self.BlockCounter) rheader.RHread(self.fpFile) #Bltr FileHeader Reading self.OffsetStartHeader=rheader.OffsetStartHeader self.RecCounter=rheader.RecCounter self.Off2StartNxtRec=rheader.Off2StartNxtRec self.Off2StartData=rheader.Off2StartData self.nProfiles=rheader.nProfiles self.nChannels=rheader.nChannels self.nHeights=rheader.nHeights self.frequency=rheader.TransmitFrec self.DualModeIndex=rheader.DualModeIndex self.pairsList =[(0,1),(0,2),(1,2)] self.dataOut.pairsList = self.pairsList self.nRdPairs=len(self.dataOut.pairsList) self.dataOut.nRdPairs = self.nRdPairs self.__firstHeigth=rheader.StartRangeSamp self.__deltaHeigth=rheader.SampResolution self.dataOut.heightList= self.__firstHeigth + numpy.array(range(self.nHeights))*self.__deltaHeigth self.dataOut.channelList = range(self.nChannels) self.dataOut.nProfiles=rheader.nProfiles self.dataOut.nIncohInt=rheader.nIncohInt self.dataOut.nCohInt=rheader.nCohInt self.dataOut.ippSeconds= 1/float(rheader.PRFhz) self.dataOut.PRF=rheader.PRFhz self.dataOut.nFFTPoints=rheader.nProfiles self.dataOut.utctime=rheader.nUtime self.dataOut.timeZone=0 self.dataOut.normFactor= self.dataOut.nProfiles*self.dataOut.nIncohInt*self.dataOut.nCohInt self.dataOut.outputInterval= self.dataOut.ippSeconds * self.dataOut.nCohInt * self.dataOut.nIncohInt * self.nProfiles self.data_output=numpy.ones([3,rheader.nHeights])*numpy.NaN print 'self.data_output', shape(self.data_output) self.dataOut.velocityX=[] self.dataOut.velocityY=[] self.dataOut.velocityV=[] '''Block Reading, the Block Data is received and Reshape is used to give it shape. ''' #Procedure to take the pointer to where the date block starts startDATA = open(self.fpFile,"rb") OffDATA= self.OffsetStartHeader + self.RecCounter*self.Off2StartNxtRec+self.Off2StartData startDATA.seek(OffDATA, os.SEEK_SET) def moving_average(x, N=2): return numpy.convolve(x, numpy.ones((N,))/N)[(N-1):] def gaus(xSamples,a,x0,sigma): return a*exp(-(xSamples-x0)**2/(2*sigma**2)) def Find(x,value): for index in range(len(x)): if x[index]==value: return index def pol2cart(rho, phi): x = rho * numpy.cos(phi) y = rho * numpy.sin(phi) return(x, y) if self.DualModeIndex==self.ReadMode: self.data_fft = numpy.fromfile( startDATA, [('complex',' 0.0001) : # # try: # popt,pcov = curve_fit(gaus,xSamples,yMean,p0=[1,meanGauss,sigma]) # # if numpy.amax(popt)>numpy.amax(yMean)*0.3: # FitGauss=gaus(xSamples,*popt) # # else: # FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean) # print 'Verificador: Dentro', Height # except RuntimeError: # # try: # for j in range(len(ySamples[1])): # yMean2=numpy.append(yMean2,numpy.average([ySamples[1,j],ySamples[2,j]])) # popt,pcov = curve_fit(gaus,xSamples,yMean2,p0=[1,meanGauss,sigma]) # FitGauss=gaus(xSamples,*popt) # print 'Verificador: Exepcion1', Height # except RuntimeError: # # try: # popt,pcov = curve_fit(gaus,xSamples,ySamples[1],p0=[1,meanGauss,sigma]) # FitGauss=gaus(xSamples,*popt) # print 'Verificador: Exepcion2', Height # except RuntimeError: # FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean) # print 'Verificador: Exepcion3', Height # else: # FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean) # #print 'Verificador: Fuera', Height # # # # Maximun=numpy.amax(yMean) # eMinus1=Maximun*numpy.exp(-1) # # HWpos=Find(FitGauss,min(FitGauss, key=lambda value:abs(value-eMinus1))) # HalfWidth= xFrec[HWpos] # GCpos=Find(FitGauss, numpy.amax(FitGauss)) # Vpos=Find(FactNorm, numpy.amax(FactNorm)) # #Vpos=numpy.sum(FactNorm)/len(FactNorm) # #Vpos=Find(FactNorm, min(FactNorm, key=lambda value:abs(value- numpy.mean(FactNorm) ))) # #print 'GCpos',GCpos, numpy.amax(FitGauss), 'HWpos',HWpos # '''****** Getting Fij ******''' # # GaussCenter=xFrec[GCpos] # if (GaussCenter<0 and HalfWidth>0) or (GaussCenter>0 and HalfWidth<0): # Fij=abs(GaussCenter)+abs(HalfWidth)+0.0000001 # else: # Fij=abs(GaussCenter-HalfWidth)+0.0000001 # # '''****** Getting Frecuency range of significant data ******''' # # Rangpos=Find(FitGauss,min(FitGauss, key=lambda value:abs(value-Maximun*0.10))) # # if Rangpos5 and len(FrecRange) 0.: # self.dataOut.velocityX=numpy.append(self.dataOut.velocityX, Vzon) #Vmag # #print 'Vmag',Vmag # else: # self.dataOut.velocityX=numpy.append(self.dataOut.velocityX, NaN) # # if abs(Vx)<100 and abs(Vx) > 0.: # self.dataOut.velocityY=numpy.append(self.dataOut.velocityY, Vmer) #Vang # #print 'Vang',Vang # else: # self.dataOut.velocityY=numpy.append(self.dataOut.velocityY, NaN) # # if abs(GaussCenter)<2: # self.dataOut.velocityV=numpy.append(self.dataOut.velocityV, xFrec[Vpos]) # # else: # self.dataOut.velocityV=numpy.append(self.dataOut.velocityV, NaN) # # # # print '********************************************' # # print 'HalfWidth ', HalfWidth # # print 'Maximun ', Maximun # # print 'eMinus1 ', eMinus1 # # print 'Rangpos ', Rangpos # # print 'GaussCenter ',GaussCenter # # print 'E01 ',E01 # # print 'N01 ',N01 # # print 'E02 ',E02 # # print 'N02 ',N02 # # print 'E12 ',E12 # # print 'N12 ',N12 # #print 'self.dataOut.velocityX ', self.dataOut.velocityX # # print 'Fij ', Fij # # print 'cC ', cC # # print 'cF ', cF # # print 'cG ', cG # # print 'cA ', cA # # print 'cB ', cB # # print 'cH ', cH # # print 'Vx ', Vx # # print 'Vy ', Vy # # print 'Vmag ', Vmag # # print 'Vang ', Vang*180/numpy.pi # # print 'PhaseSlope ',PhaseSlope[0] # # print 'PhaseSlope ',PhaseSlope[1] # # print 'PhaseSlope ',PhaseSlope[2] # # print '********************************************' # #print 'data_output',shape(self.dataOut.velocityX), shape(self.dataOut.velocityY) # # #print 'self.dataOut.velocityX', len(self.dataOut.velocityX) # #print 'self.dataOut.velocityY', len(self.dataOut.velocityY) # #print 'self.dataOut.velocityV', self.dataOut.velocityV # # self.data_output[0]=numpy.array(self.dataOut.velocityX) # self.data_output[1]=numpy.array(self.dataOut.velocityY) # self.data_output[2]=numpy.array(self.dataOut.velocityV) # # prin= self.data_output[0][~numpy.isnan(self.data_output[0])] # print ' ' # print 'VmagAverage',numpy.mean(prin) # print ' ' # # plt.figure(5) # # plt.subplot(211) # # plt.plot(self.dataOut.velocityX,'yo:') # # plt.subplot(212) # # plt.plot(self.dataOut.velocityY,'yo:') # # # plt.figure(1) # # # plt.subplot(121) # # # plt.plot(xFrec,ySamples[0],'k',label='Ch0') # # # plt.plot(xFrec,ySamples[1],'g',label='Ch1') # # # plt.plot(xFrec,ySamples[2],'r',label='Ch2') # # # plt.plot(xFrec,FitGauss,'yo:',label='fit') # # # plt.legend() # # plt.title('DATOS A ALTURA DE 2850 METROS') # # # # plt.xlabel('Frecuencia (KHz)') # # plt.ylabel('Magnitud') # # # plt.subplot(122) # # # plt.title('Fit for Time Constant') # # #plt.plot(xFrec,zline) # # #plt.plot(xFrec,SmoothSPC,'g') # # plt.plot(xFrec,FactNorm) # # plt.axis([-4, 4, 0, 0.15]) # # # plt.xlabel('SelfSpectra KHz') # # # # plt.figure(10) # # # plt.subplot(121) # # plt.plot(xFrec,ySamples[0],'b',label='Ch0') # # plt.plot(xFrec,ySamples[1],'y',label='Ch1') # # plt.plot(xFrec,ySamples[2],'r',label='Ch2') # # # plt.plot(xFrec,FitGauss,'yo:',label='fit') # # plt.legend() # # plt.title('SELFSPECTRA EN CANALES') # # # # plt.xlabel('Frecuencia (KHz)') # # plt.ylabel('Magnitud') # # # plt.subplot(122) # # # plt.title('Fit for Time Constant') # # #plt.plot(xFrec,zline) # # #plt.plot(xFrec,SmoothSPC,'g') # # # plt.plot(xFrec,FactNorm) # # # plt.axis([-4, 4, 0, 0.15]) # # # plt.xlabel('SelfSpectra KHz') # # # # plt.figure(9) # # # # # # plt.title('DATOS SUAVIZADOS') # # plt.xlabel('Frecuencia (KHz)') # # plt.ylabel('Magnitud') # # plt.plot(xFrec,SmoothSPC,'g') # # # # #plt.plot(xFrec,FactNorm) # # plt.axis([-4, 4, 0, 0.15]) # # # plt.xlabel('SelfSpectra KHz') # # # # # plt.figure(2) # # # #plt.subplot(121) # # plt.plot(xFrec,yMean,'r',label='Mean SelfSpectra') # # plt.plot(xFrec,FitGauss,'yo:',label='Ajuste Gaussiano') # # # plt.plot(xFrec[Rangpos],FitGauss[Find(FitGauss,min(FitGauss, key=lambda value:abs(value-Maximun*0.1)))],'bo') # # # #plt.plot(xFrec,phase) # # # plt.xlabel('Suavizado, promediado KHz') # # plt.title('SELFSPECTRA PROMEDIADO') # # # #plt.subplot(122) # # # #plt.plot(xSamples,zline) # # plt.xlabel('Frecuencia (KHz)') # # plt.ylabel('Magnitud') # # plt.legend() # # # # # # plt.figure(3) # # # plt.subplot(311) # # # #plt.plot(xFrec,phase[0]) # # # plt.plot(xFrec,phase[0],'g') # # # plt.subplot(312) # # # plt.plot(xFrec,phase[1],'g') # # # plt.subplot(313) # # # plt.plot(xFrec,phase[2],'g') # # # #plt.plot(xFrec,phase[2]) # # # # # # plt.figure(4) # # # # # # plt.plot(xSamples,coherence[0],'b') # # # plt.plot(xSamples,coherence[1],'r') # # # plt.plot(xSamples,coherence[2],'g') # # plt.show() # # # # # # plt.clf() # # # plt.cla() # # # plt.close() # # print ' ' self.BlockCounter+=2 else: self.fileSelector+=1 self.BlockCounter=0 print "Next File"