From f9eef645eb5337a2239e3e2b0028ef3b06d39d40 2022-04-29 22:31:17 From: Roberto Flores Date: 2022-04-29 22:31:17 Subject: [PATCH] New RHI Plot and Block 360 stores more parameters --- diff --git a/schainpy/model/graphics/jroplot_base.py b/schainpy/model/graphics/jroplot_base.py index 231e57a..140116d 100644 --- a/schainpy/model/graphics/jroplot_base.py +++ b/schainpy/model/graphics/jroplot_base.py @@ -254,6 +254,7 @@ class Plot(Operation): self.data = PlotterData(self.CODE, self.exp_code, self.localtime) self.ang_min = kwargs.get('ang_min', None) self.ang_max = kwargs.get('ang_max', None) + self.mode = kwargs.get('mode', None) if self.server: @@ -434,9 +435,15 @@ class Plot(Operation): self.time_label), size=8) else: - ax.set_title('{}'.format(self.titles[n]), size=8) - ax.set_ylim(0, 90) - ax.set_yticks(numpy.arange(0, 90, 20)) + #ax.set_title('{}'.format(self.titles[n]), size=8) + ax.set_title('{} {} {}'.format( + self.titles[n], + self.getDateTime(self.data.max_time).strftime( + '%Y-%m-%d %H:%M:%S'), + self.time_label), + size=8) + ax.set_ylim(0, self.ymax) + #ax.set_yticks(numpy.arange(0, self.ymax, 20)) ax.yaxis.labelpad = 40 if self.firsttime: diff --git a/schainpy/model/graphics/jroplot_parameters.py b/schainpy/model/graphics/jroplot_parameters.py index 0dc1684..8eab752 100644 --- a/schainpy/model/graphics/jroplot_parameters.py +++ b/schainpy/model/graphics/jroplot_parameters.py @@ -2376,3 +2376,147 @@ class WeatherRHI_vRF3_Plot(Plot): plt.text(1.0, 1.05, 'Elevacion '+str(thisDatetime)+" Step "+str(self.ini)+ " Azi: "+str(round(self.res_azi,2)), transform=caax.transAxes, va='bottom',ha='right') print("***************************self.ini****************************",self.ini) self.ini= self.ini+1 + +class WeatherRHI_vRF4_Plot(Plot): + CODE = 'weather' + plot_name = 'weather' + #plot_type = 'rhistyle' + buffering = False + data_ele_tmp = None + + def setup(self): + + self.ncols = 1 + self.nrows = 1 + self.nplots= 1 + self.ylabel= 'Range [Km]' + self.titles= ['Weather'] + self.polar = True + if self.channels is not None: + self.nplots = len(self.channels) + self.nrows = len(self.channels) + else: + self.nplots = self.data.shape(self.CODE)[0] + self.nrows = self.nplots + self.channels = list(range(self.nplots)) + #print("JERE") + #exit(1) + #print("channels",self.channels) + #print("que saldra", self.data.shape(self.CODE)[0]) + #self.titles = ['{} Channel {}'.format(self.CODE.upper(), x) for x in range(self.nrows)] + + #print("self.titles",self.titles) + if self.CODE == 'Power': + self.cb_label = r'Power (dB)' + elif self.CODE == 'Doppler': + self.cb_label = r'Velocity (m/s)' + self.colorbar=True + self.width =8 + self.height =8 + self.ini =0 + self.len_azi =0 + self.buffer_ini = None + self.buffer_ele = None + self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.08}) + self.flag =0 + self.indicador= 0 + self.last_data_ele = None + self.val_mean = None + + def update(self, dataOut): + + if self.mode == 'Power': + self.CODE = 'Power' + elif self.mode == 'Doppler': + self.CODE = 'Doppler' + + data = {} + meta = {} + if hasattr(dataOut, 'dataPP_POWER'): + factor = 1 + if hasattr(dataOut, 'nFFTPoints'): + factor = dataOut.normFactor + + if self.CODE == 'Power': + data[self.CODE] = 10*numpy.log10(dataOut.data_360_Power/(factor)) + elif self.CODE == 'Doppler': + data[self.CODE] = dataOut.data_360_Velocity/(factor) + + data['azi'] = dataOut.data_azi + data['ele'] = dataOut.data_ele + + return data, meta + + def plot(self): + thisDatetime = datetime.datetime.utcfromtimestamp(self.data.times[-1]).strftime('%Y-%m-%d %H:%M:%S') + data = self.data[-1] + r = self.data.yrange + delta_height = r[1]-r[0] + r_mask = numpy.where(r>=0)[0] + self.r_mask =r_mask + r = numpy.arange(len(r_mask))*delta_height + self.y = 2*r + res = 1 + ang_max = self.ang_max + ang_min = self.ang_min + var_ang =ang_max - ang_min + step = (int(var_ang)/(res*data[self.CODE].shape[0])) + + z = data[self.CODE][self.channels[0]][:,r_mask] + + #print(z[2,:]) + self.titles = [] + + #exit(1) + + if self.CODE == 'Power': + cmap = 'jet' + elif self.CODE == 'Doppler': + cmap = 'RdBu' + + self.ymax = self.ymax if self.ymax else numpy.nanmax(r) + self.ymin = self.ymin if self.ymin else numpy.nanmin(r) + self.zmax = self.zmax if self.zmax else numpy.nanmax(z) + self.zmin = self.zmin if self.zmin else numpy.nanmin(z) + + #plt.clf() + subplots = [121, 122] + + r, theta = numpy.meshgrid(r, numpy.radians(data['ele']) ) + + points_cb = 200 + mylevs_cbar = list(numpy.linspace(self.zmin,self.zmax,points_cb)) #niveles de la barra de colores + + for i,ax in enumerate(self.axes): + + if ax.firsttime: + ax.set_xlim(numpy.radians(self.ang_min),numpy.radians(self.ang_max)) + ax.plt = ax.contourf(theta, r, z, points_cb, cmap=cmap, vmin=self.zmin, vmax=self.zmax, levels=mylevs_cbar) + #print(ax.plt) + #exit(1) + ''' + self.figures[-1].colorbar(plt, orientation="vertical", fraction=0.025, pad=0.07) + print(self.figures[0]) + print(self.figures) + print(plt) + print(ax) + exit(1) + ''' + + else: + ax.set_xlim(numpy.radians(self.ang_min),numpy.radians(self.ang_max)) + ax.plt = ax.contourf(theta, r, z, points_cb, cmap=cmap, vmin=self.zmin, vmax=self.zmax, levels=mylevs_cbar) + #self.figures[0].colorbar(plt, orientation="vertical", fraction=0.025, pad=0.07) + + #print(self.titles) + if len(self.channels) !=1: + self.titles = ['{} Azi: {} Channel {}'.format(self.CODE.upper(), str(round(numpy.mean(data['azi']),2)), x) for x in range(self.nrows)] + else: + self.titles = ['{} Azi: {} Channel {}'.format(self.CODE.upper(), str(round(numpy.mean(data['azi']),2)), self.channels[0])] + #self.titles.append('Azi: {}'.format(str(round(numpy.mean(data['azi']),2)))) + #self.titles.append(str(round(numpy.mean(data['azi']),2))) + #print(self.titles) + #plt.text(1.0, 1.05, str(thisDatetime)+ " Azi: "+str(round(numpy.mean(data['azi']),2)), transform=caax.transAxes, va='bottom',ha='right') + #print("***************************self.ini****************************",self.ini) + #self.figures[-1].colorbar(plt, orientation="vertical", fraction=0.025, pad=0.07) + #self.ini= self.ini+1 diff --git a/schainpy/model/io/jroIO_param.py b/schainpy/model/io/jroIO_param.py index 9d5e3d0..6c7890e 100644 --- a/schainpy/model/io/jroIO_param.py +++ b/schainpy/model/io/jroIO_param.py @@ -394,11 +394,22 @@ class HDFWriter(Operation): flag = True return flag + def generalFlag_vRF(self): + ####rint("GENERALFLAG") + + try: + self.dataOut.flagBlock360Done + return self.dataOut.flagBlock360Done + except: + return 0 + + def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None,type_data=None,**kwargs): self.path = path self.blocksPerFile = blocksPerFile self.metadataList = metadataList self.dataList = [s.strip() for s in dataList] + self.setType = setType if self.mode == "weather": self.setType = "weather" #---------------------------------------- @@ -624,7 +635,7 @@ class HDFWriter(Operation): return def writeData(self, fp): - + print("writing data") if self.description: if 'Data' in self.description: grp = fp.create_group('Data') @@ -671,7 +682,7 @@ class HDFWriter(Operation): def putData(self): ###print("**************************PUT DATA***************************************************") - if (self.blockIndex == self.blocksPerFile) or self.timeFlag() or self.generalFlag(): + if (self.blockIndex == self.blocksPerFile) or self.timeFlag():# or self.generalFlag_vRF(): self.closeFile() self.setNextFile() diff --git a/schainpy/model/proc/jroproc_base.py b/schainpy/model/proc/jroproc_base.py index 97b1906..3f1636e 100644 --- a/schainpy/model/proc/jroproc_base.py +++ b/schainpy/model/proc/jroproc_base.py @@ -4,6 +4,7 @@ must be used in plotting and writing operations to allow to run as an external process. ''' +import os import inspect import zmq import time @@ -13,6 +14,7 @@ from threading import Thread from multiprocessing import Process, Queue from schainpy.utils import log +QUEUE_SIZE = int(os.environ.get('QUEUE_MAX_SIZE', '10')) class ProcessingUnit(object): ''' @@ -178,7 +180,7 @@ def MPDecorator(BaseClass): self.start_time = time.time() self.err_queue = args[3] - self.queue = Queue(maxsize=1) + self.queue = Queue(maxsize=QUEUE_SIZE) self.myrun = BaseClass.run def run(self): diff --git a/schainpy/model/proc/jroproc_parameters.py b/schainpy/model/proc/jroproc_parameters.py index 8ae97ec..009f7b5 100755 --- a/schainpy/model/proc/jroproc_parameters.py +++ b/schainpy/model/proc/jroproc_parameters.py @@ -4054,7 +4054,7 @@ class PedestalInformation(Operation): dt = datetime.datetime.utcfromtimestamp(timestamp) path = os.path.join(self.path, dt.strftime('%Y-%m-%dT%H-00-00')) - + if not os.path.exists(path): return False, False fileList = glob.glob(os.path.join(path, '*.h5')) @@ -4069,7 +4069,7 @@ class PedestalInformation(Operation): def find_next_file(self): while True: - file_size = len(self.fp['Data']['utc']) + file_size = len(self.fp['Data']['utc']) if self.utctime < self.utcfile+file_size*self.interval: break self.utcfile += file_size*self.interval @@ -4078,14 +4078,14 @@ class PedestalInformation(Operation): self.filename = os.path.join(path, 'pos@{}.000.h5'.format(int(self.utcfile))) if not os.path.exists(self.filename): log.warning('Waiting for position files...', self.name) - + if not os.path.exists(self.filename): - + raise IOError('No new position files found in {}'.format(path)) self.fp.close() self.fp = h5py.File(self.filename, 'r') log.log('Opening file: {}'.format(self.filename), self.name) - + def get_values(self): index = int((self.utctime-self.utcfile)/self.interval) @@ -4098,7 +4098,7 @@ class PedestalInformation(Operation): self.samples = samples self.interval = interval self.utcfile, self.filename = self.find_file(dataOut.utctime) - + if not self.filename: log.error('No position files found in {}'.format(path), self.name) raise IOError('No position files found in {}'.format(path)) @@ -4107,15 +4107,15 @@ class PedestalInformation(Operation): self.fp = h5py.File(self.filename, 'r') def run(self, dataOut, path, conf=None, samples=1500, interval=0.04, wr_exp=None): - + if not self.isConfig: self.setup(dataOut, path, conf, samples, interval, wr_exp) self.isConfig = True - + self.utctime = dataOut.utctime - + self.find_next_file() - + az, el = self.get_values() dataOut.flagNoData = False @@ -4509,7 +4509,7 @@ class Block360_vRF2(Operation): __nch = 0 __nHeis = 0 index = 0 - mode = 0 + mode = None def __init__(self,**kwargs): Operation.__init__(self,**kwargs) @@ -4534,28 +4534,24 @@ class Block360_vRF2(Operation): self.__buffer = [] self.__buffer2 = [] self.__buffer3 = [] + self.__buffer4 = [] def putData(self,data,mode): ''' Add a profile to he __buffer and increase in one the __profiel Index ''' - #print("line 4049",data.dataPP_POW.shape,data.dataPP_POW[:10]) - #print("line 4049",data.azimuth.shape,data.azimuth) + if self.mode==0: self.__buffer.append(data.dataPP_POWER)# PRIMER MOMENTO if self.mode==1: self.__buffer.append(data.data_pow) - #print("me casi",self.index,data.azimuth[self.index]) - #print(self.__profIndex, self.index , data.azimuth[self.index] ) - #print("magic",data.profileIndex) - #print(data.azimuth[self.index]) - #print("index",self.index) - #####self.__buffer2[self.__profIndex] = data.azimuth[self.index] + self.__buffer4.append(data.dataPP_DOP) + self.__buffer2.append(data.azimuth) self.__buffer3.append(data.elevation) self.__profIndex += 1 - #print("q pasa") + return numpy.array(self.__buffer3) #················· Remove DC··································· def pushData(self,data): @@ -4563,98 +4559,233 @@ class Block360_vRF2(Operation): Return the PULSEPAIR and the profiles used in the operation Affected : self.__profileIndex ''' - #print("pushData") - data_360 = numpy.array(self.__buffer).transpose(1,0,2) + data_360_Power = numpy.array(self.__buffer).transpose(1,0,2) + data_360_Velocity = numpy.array(self.__buffer4).transpose(1,0,2) data_p = numpy.array(self.__buffer2) data_e = numpy.array(self.__buffer3) n = self.__profIndex - self.__buffer = [] + self.__buffer = [] + self.__buffer4 = [] self.__buffer2 = [] self.__buffer3 = [] self.__profIndex = 0 - #print("pushData") - return data_360,n,data_p,data_e + return data_360_Power,data_360_Velocity,n,data_p,data_e def byProfiles(self,dataOut): self.__dataReady = False - data_360 = None + data_360_Power = [] + data_360_Velocity = [] data_p = None data_e = None - #print("dataOu",dataOut.dataPP_POW) elevations = self.putData(data=dataOut,mode = self.mode) - ##### print("profIndex",self.__profIndex) - if self.__profIndex > 1: case_flag = self.checkcase(elevations) if case_flag == 0: #Subida - #Se borra el dato anterior para liberar buffer y comparar el dato actual con el siguiente + if len(self.__buffer) == 2: #Cuando está de subida + #Se borra el dato anterior para liberar buffer y comparar el dato actual con el siguiente self.__buffer.pop(0) #Erase first data self.__buffer2.pop(0) self.__buffer3.pop(0) + self.__buffer4.pop(0) self.__profIndex -= 1 else: #Cuando ha estado de bajada y ha vuelto a subir - #print("else",self.__buffer3) + #Se borra el último dato self.__buffer.pop() #Erase last data self.__buffer2.pop() self.__buffer3.pop() - data_360,n,data_p,data_e = self.pushData(data=dataOut) - #print(data_360.shape) - #print(data_e.shape) - #exit(1) + self.__buffer4.pop() + data_360_Power,data_360_Velocity,n,data_p,data_e = self.pushData(data=dataOut) + self.__dataReady = True - ''' - elif elevations[-1]<0.: - if len(self.__buffer) == 2: + + return data_360_Power,data_360_Velocity,data_p,data_e + + + def blockOp(self, dataOut, datatime= None): + if self.__initime == None: + self.__initime = datatime + data_360_Power,data_360_Velocity,data_p,data_e = self.byProfiles(dataOut) + self.__lastdatatime = datatime + + avgdatatime = self.__initime + if self.n==1: + avgdatatime = datatime + deltatime = datatime - self.__lastdatatime + self.__initime = datatime + return data_360_Power,data_360_Velocity,avgdatatime,data_p,data_e + + def checkcase(self,data_ele): + #print(data_ele) + start = data_ele[-2] + end = data_ele[-1] + diff_angle = (end-start) + len_ang=len(data_ele) + + if diff_angle > 0: #Subida + return 0 + + def run(self, dataOut,mode='Power',**kwargs): + #print("BLOCK 360 HERE WE GO MOMENTOS") + #print("Block 360") + dataOut.mode = mode + + if not self.isConfig: + self.setup(dataOut = dataOut ,mode= mode ,**kwargs) + self.isConfig = True + + + data_360_Power, data_360_Velocity, avgdatatime,data_p,data_e = self.blockOp(dataOut, dataOut.utctime) + + + dataOut.flagNoData = True + + + if self.__dataReady: + dataOut.data_360_Power = data_360_Power # S + dataOut.data_360_Velocity = data_360_Velocity + dataOut.data_azi = data_p + dataOut.data_ele = data_e + dataOut.utctime = avgdatatime + dataOut.flagNoData = False + + return dataOut + +class Block360_vRF3(Operation): + ''' + ''' + isConfig = False + __profIndex = 0 + __initime = None + __lastdatatime = None + __buffer = None + __dataReady = False + n = None + __nch = 0 + __nHeis = 0 + index = 0 + mode = None + + def __init__(self,**kwargs): + Operation.__init__(self,**kwargs) + + def setup(self, dataOut, n = None, mode = None): + ''' + n= Numero de PRF's de entrada + ''' + self.__initime = None + self.__lastdatatime = 0 + self.__dataReady = False + self.__buffer = 0 + self.__buffer_1D = 0 + #self.__profIndex = 0 + self.index = 0 + self.__nch = dataOut.nChannels + self.__nHeis = dataOut.nHeights + + self.mode = mode + #print("self.mode",self.mode) + #print("nHeights") + self.__buffer = [] + self.__buffer2 = [] + self.__buffer3 = [] + self.__buffer4 = [] + + def putData(self,data,mode): + ''' + Add a profile to he __buffer and increase in one the __profiel Index + ''' + + if self.mode==0: + self.__buffer.append(data.dataPP_POWER)# PRIMER MOMENTO + if self.mode==1: + self.__buffer.append(data.data_pow) + + self.__buffer4.append(data.dataPP_DOP) + + self.__buffer2.append(data.azimuth) + self.__buffer3.append(data.elevation) + self.__profIndex += 1 + + return numpy.array(self.__buffer3) #················· Remove DC··································· + + def pushData(self,data): + ''' + Return the PULSEPAIR and the profiles used in the operation + Affected : self.__profileIndex + ''' + + data_360_Power = numpy.array(self.__buffer).transpose(1,0,2) + data_360_Velocity = numpy.array(self.__buffer4).transpose(1,0,2) + data_p = numpy.array(self.__buffer2) + data_e = numpy.array(self.__buffer3) + n = self.__profIndex + + self.__buffer = [] + self.__buffer4 = [] + self.__buffer2 = [] + self.__buffer3 = [] + self.__profIndex = 0 + return data_360_Power,data_360_Velocity,n,data_p,data_e + + + def byProfiles(self,dataOut): + + self.__dataReady = False + data_360_Power = [] + data_360_Velocity = [] + data_p = None + data_e = None + + elevations = self.putData(data=dataOut,mode = self.mode) + + if self.__profIndex > 1: + case_flag = self.checkcase(elevations) + + if case_flag == 0: #Subida + + if len(self.__buffer) == 2: #Cuando está de subida + #Se borra el dato anterior para liberar buffer y comparar el dato actual con el siguiente self.__buffer.pop(0) #Erase first data self.__buffer2.pop(0) self.__buffer3.pop(0) + self.__buffer4.pop(0) self.__profIndex -= 1 - else: + else: #Cuando ha estado de bajada y ha vuelto a subir + #Se borra el último dato self.__buffer.pop() #Erase last data self.__buffer2.pop() self.__buffer3.pop() - data_360,n,data_p,data_e = self.pushData(data=dataOut) - self.__dataReady = True - ''' - + self.__buffer4.pop() + data_360_Power,data_360_Velocity,n,data_p,data_e = self.pushData(data=dataOut) - ''' - if self.__profIndex == self.n: - data_360,n,data_p,data_e = self.pushData(data=dataOut) - self.__dataReady = True - ''' + self.__dataReady = True - return data_360,data_p,data_e + return data_360_Power,data_360_Velocity,data_p,data_e def blockOp(self, dataOut, datatime= None): if self.__initime == None: self.__initime = datatime - data_360,data_p,data_e = self.byProfiles(dataOut) + data_360_Power,data_360_Velocity,data_p,data_e = self.byProfiles(dataOut) self.__lastdatatime = datatime - if data_360 is None: - return None, None,None,None - - avgdatatime = self.__initime if self.n==1: avgdatatime = datatime deltatime = datatime - self.__lastdatatime self.__initime = datatime - #print(data_360.shape,avgdatatime,data_p.shape) - return data_360,avgdatatime,data_p,data_e + return data_360_Power,data_360_Velocity,avgdatatime,data_p,data_e def checkcase(self,data_ele): - print(data_ele) + #print(data_ele) start = data_ele[-2] end = data_ele[-1] diff_angle = (end-start) @@ -4663,46 +4794,28 @@ class Block360_vRF2(Operation): if diff_angle > 0: #Subida return 0 - def run(self, dataOut,n = None,mode=None,**kwargs): + def run(self, dataOut,mode='Power',**kwargs): #print("BLOCK 360 HERE WE GO MOMENTOS") - print("Block 360") + #print("Block 360") + dataOut.mode = mode - #exit(1) if not self.isConfig: - - print(n) self.setup(dataOut = dataOut ,mode= mode ,**kwargs) - ####self.index = 0 - #print("comova",self.isConfig) self.isConfig = True - ####if self.index==dataOut.azimuth.shape[0]: - #### self.index=0 - - data_360, avgdatatime,data_p,data_e = self.blockOp(dataOut, dataOut.utctime) + data_360_Power, data_360_Velocity, avgdatatime,data_p,data_e = self.blockOp(dataOut, dataOut.utctime) dataOut.flagNoData = True + if self.__dataReady: - dataOut.data_360 = data_360 # S - #print("DATA 360") - #print(dataOut.data_360) - #print("---------------------------------------------------------------------------------") - print("---------------------------DATAREADY---------------------------------------------") - #print("---------------------------------------------------------------------------------") - #print("data_360",dataOut.data_360.shape) - print(data_e) - #exit(1) + dataOut.data_360_Power = data_360_Power # S + dataOut.data_360_Velocity = data_360_Velocity dataOut.data_azi = data_p dataOut.data_ele = data_e - ###print("azi: ",dataOut.data_azi) - #print("ele: ",dataOut.data_ele) - #print("jroproc_parameters",data_p[0],data_p[-1])#,data_360.shape,avgdatatime) dataOut.utctime = avgdatatime - - - dataOut.flagNoData = False + return dataOut