diff --git a/schainpy/__init__.py b/schainpy/__init__.py index d46a2d0..4654380 100644 --- a/schainpy/__init__.py +++ b/schainpy/__init__.py @@ -4,4 +4,4 @@ Created on Feb 7, 2012 @author $Author$ @version $Id$ ''' -__version__ = "2.3" +__version__ = '2.3' diff --git a/schainpy/model/data/BLTRheaderIO.py b/schainpy/model/data/BLTRheaderIO.py new file mode 100644 index 0000000..69e0d25 --- /dev/null +++ b/schainpy/model/data/BLTRheaderIO.py @@ -0,0 +1,404 @@ +''' + +$Author: murco $ +$Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $ +''' +import sys +import numpy +import copy +import datetime +from __builtin__ import None + +SPEED_OF_LIGHT = 299792458 +SPEED_OF_LIGHT = 3e8 + +FILE_STRUCTURE = numpy.dtype([ #HEADER 48bytes + ('FileMgcNumber',' vertical) + ('AntennaCoord',' endFp: + sys.stderr.write("Warning %s: Size value read from System Header is lower than it has to be\n" %fp.name) + return 0 + + if fp.tell() < endFp: + sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" %fp.name) + return 0 + + return 1 + + def write(self, fp): + + headerTuple = (self.RecMgcNumber, + self.RecCounter, + self.Off2StartNxtRec, + self.EpTimeStamp, + self.msCompTimeStamp, + self.ExpTagName, + self.ExpComment, + self.SiteLatDegrees, + self.SiteLongDegrees, + self.RTCgpsStatus, + self.TransmitFrec, + self.ReceiveFrec, + self.FirstOsciFrec, + self.Polarisation, + self.ReceiverFiltSett, + self.nModesInUse, + self.DualModeIndex, + self.DualModeRange, + self.nDigChannels, + self.SampResolution, + self.nRangeGatesSamp, + self.StartRangeSamp, + self.PRFhz, + self.Integrations, + self.nDataPointsTrsf, + self.nReceiveBeams, + self.nSpectAverages, + self.FFTwindowingInd, + self.BeamAngleAzim, + self.BeamAngleZen, + self.AntennaCoord, + self.RecPhaseCalibr, + self.RecAmpCalibr, + self.ReceiverGaindB) + +# self.size,self.nSamples, +# self.nProfiles, +# self.nChannels, +# self.adcResolution, +# self.pciDioBusWidth + + header = numpy.array(headerTuple,RECORD_STRUCTURE) + header.tofile(fp) + + return 1 + + +def get_dtype_index(numpy_dtype): + + index = None + + for i in range(len(NUMPY_DTYPE_LIST)): + if numpy_dtype == NUMPY_DTYPE_LIST[i]: + index = i + break + + return index + +def get_numpy_dtype(index): + + #dtype4 = numpy.dtype([('real','= self.nplots: + break + + self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1) + + if showprofile: + self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1) + + counter += 1 + + def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True, + xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None, + save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1, + server=None, folder=None, username=None, password=None, + ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False, + xaxis="frequency", colormap='jet', normFactor=None , GauSelector = 1): + + """ + + Input: + dataOut : + id : + wintitle : + channelList : + showProfile : + xmin : None, + xmax : None, + ymin : None, + ymax : None, + zmin : None, + zmax : None + """ + if realtime: + if not(isRealtime(utcdatatime = dataOut.utctime)): + print 'Skipping this plot function' + return + + if channelList == None: + channelIndexList = dataOut.channelIndexList + else: + channelIndexList = [] + for channel in channelList: + if channel not in dataOut.channelList: + raise ValueError, "Channel %d is not in dataOut.channelList" %channel + channelIndexList.append(dataOut.channelList.index(channel)) + +# if normFactor is None: +# factor = dataOut.normFactor +# else: +# factor = normFactor + if xaxis == "frequency": + x = dataOut.spc_range[0] + xlabel = "Frequency (kHz)" + + elif xaxis == "time": + x = dataOut.spc_range[1] + xlabel = "Time (ms)" + + else: + x = dataOut.spc_range[2] + xlabel = "Velocity (m/s)" + + ylabel = "Range (Km)" + + y = dataOut.getHeiRange() + + z = dataOut.GauSPC[:,GauSelector,:,:] #GauSelector] #dataOut.data_spc/factor + print 'GausSPC', z[0,32,10:40] + z = numpy.where(numpy.isfinite(z), z, numpy.NAN) + zdB = 10*numpy.log10(z) + + avg = numpy.average(z, axis=1) + avgdB = 10*numpy.log10(avg) + + noise = dataOut.spc_noise + noisedB = 10*numpy.log10(noise) + + thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0]) + title = wintitle + " Spectra" + if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)): + title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith) + + if not self.isConfig: + + nplots = len(channelIndexList) + + self.setup(id=id, + nplots=nplots, + wintitle=wintitle, + showprofile=showprofile, + show=show) + + if xmin == None: xmin = numpy.nanmin(x) + if xmax == None: xmax = numpy.nanmax(x) + if ymin == None: ymin = numpy.nanmin(y) + if ymax == None: ymax = numpy.nanmax(y) + if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3 + if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3 + + self.FTP_WEI = ftp_wei + self.EXP_CODE = exp_code + self.SUB_EXP_CODE = sub_exp_code + self.PLOT_POS = plot_pos + + self.isConfig = True + + self.setWinTitle(title) + + for i in range(self.nplots): + index = channelIndexList[i] + str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S")) + title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[index], noisedB[index], str_datetime) + if len(dataOut.beam.codeList) != 0: + title = "Ch%d:%4.2fdB,%2.2f,%2.2f:%s" %(dataOut.channelList[index], noisedB[index], dataOut.beam.azimuthList[index], dataOut.beam.zenithList[index], str_datetime) + + axes = self.axesList[i*self.__nsubplots] + axes.pcolor(x, y, zdB[index,:,:], + xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax, + xlabel=xlabel, ylabel=ylabel, title=title, colormap=colormap, + ticksize=9, cblabel='') + + if self.__showprofile: + axes = self.axesList[i*self.__nsubplots +1] + axes.pline(avgdB[index,:], y, + xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax, + xlabel='dB', ylabel='', title='', + ytick_visible=False, + grid='x') + + noiseline = numpy.repeat(noisedB[index], len(y)) + axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2) + + self.draw() + + if figfile == None: + str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") + name = str_datetime + if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)): + name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith) + figfile = self.getFilename(name) + + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime) + + + class MomentsPlot(Figure): isConfig = None @@ -445,10 +656,9 @@ class WindProfilerPlot(Figure): # tmin = None # tmax = None - - x = dataOut.getTimeRange1(dataOut.outputInterval) - y = dataOut.heightList - z = dataOut.data_output.copy() + x = dataOut.getTimeRange1(dataOut.paramInterval) + y = dataOut.heightList + z = dataOut.data_output.copy() nplots = z.shape[0] #Number of wind dimensions estimated nplotsw = nplots @@ -558,7 +768,7 @@ class WindProfilerPlot(Figure): thisDatetime=thisDatetime, update_figfile=update_figfile) - if dataOut.ltctime + dataOut.outputInterval >= self.xmax: + if dataOut.ltctime + dataOut.paramInterval >= self.xmax: self.counter_imagwr = wr_period self.isConfig = False update_figfile = True @@ -635,12 +845,12 @@ class ParametersPlot(Figure): counter += 1 - def run(self, dataOut, id, wintitle="", channelList=None, paramIndex = 0, colormap=True, + def run(self, dataOut, id, wintitle="", channelList=None, paramIndex = 0, colormap="jet", xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None, timerange=None, showSNR=False, SNRthresh = -numpy.inf, SNRmin=None, SNRmax=None, save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True, server=None, folder=None, username=None, password=None, - ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0): + ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, HEIGHT=None): """ Input: @@ -656,12 +866,11 @@ class ParametersPlot(Figure): zmin : None, zmax : None """ - - if colormap: - colormap="jet" - else: - colormap="RdBu_r" - + + if HEIGHT is not None: + self.HEIGHT = HEIGHT + + if not isTimeInHourRange(dataOut.datatime, xmin, xmax): return @@ -906,25 +1115,23 @@ class Parameters1Plot(Figure): x = dataOut.getTimeRange1(dataOut.paramInterval) y = dataOut.heightList - z = data_param[channelIndexList,parameterIndex,:].copy() - zRange = dataOut.abscissaList -# nChannels = z.shape[0] #Number of wind dimensions estimated -# thisDatetime = dataOut.datatime + if dataOut.data_param.ndim == 3: + z = dataOut.data_param[channelIndexList,parameterIndex,:] + else: + z = dataOut.data_param[channelIndexList,:] if dataOut.data_SNR is not None: - SNRarray = dataOut.data_SNR[channelIndexList,:] - SNRdB = 10*numpy.log10(SNRarray) -# SNRavgdB = 10*numpy.log10(SNRavg) - ind = numpy.where(SNRdB < 10**(SNRthresh/10)) - z[ind] = numpy.nan + if dataOut.data_SNR.ndim == 2: + SNRavg = numpy.average(dataOut.data_SNR, axis=0) + else: + SNRavg = dataOut.data_SNR + SNRdB = 10*numpy.log10(SNRavg) thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0]) title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y")) xlabel = "" - ylabel = "Range (Km)" - - if (SNR and not onlySNR): nplots = 2*nplots + ylabel = "Range (Km)" if onlyPositive: colormap = "jet" @@ -943,8 +1150,8 @@ class Parameters1Plot(Figure): if ymin == None: ymin = numpy.nanmin(y) if ymax == None: ymax = numpy.nanmax(y) - if zmin == None: zmin = numpy.nanmin(zRange) - if zmax == None: zmax = numpy.nanmax(zRange) + if zmin == None: zmin = numpy.nanmin(z) + if zmax == None: zmax = numpy.nanmax(z) if SNR: if SNRmin == None: SNRmin = numpy.nanmin(SNRdB) @@ -994,19 +1201,18 @@ class Parameters1Plot(Figure): xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap, ticksize=9, cblabel=zlabel, cbsize="1%") - if SNR: - title = "Channel %d Signal Noise Ratio (SNR): %s" %(channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S")) - axes = self.axesList[(j)*self.__nsubplots] - if not onlySNR: - axes = self.axesList[(j + 1)*self.__nsubplots] - - axes = self.axesList[(j + nGraphsByChannel-1)] + if SNR: + title = "Channel %d Signal Noise Ratio (SNR): %s" %(channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S")) + axes = self.axesList[(j)*self.__nsubplots] + if not onlySNR: + axes = self.axesList[(j + 1)*self.__nsubplots] - z1 = SNRdB[i,:].reshape((1,-1)) - axes.pcolorbuffer(x, y, z1, - xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax, - xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap="jet", - ticksize=9, cblabel=zlabel, cbsize="1%") + axes = self.axesList[(j + nGraphsByChannel-1)] + z1 = SNRdB.reshape((1,-1)) + axes.pcolorbuffer(x, y, z1, + xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax, + xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap="jet", + ticksize=9, cblabel=zlabel, cbsize="1%") diff --git a/schainpy/model/graphics/jroplot_spectra.py b/schainpy/model/graphics/jroplot_spectra.py index d2b11f1..8fb55c4 100644 --- a/schainpy/model/graphics/jroplot_spectra.py +++ b/schainpy/model/graphics/jroplot_spectra.py @@ -87,7 +87,7 @@ class SpectraPlot(Figure): save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1, server=None, folder=None, username=None, password=None, ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False, - xaxis="velocity", **kwargs): + xaxis="frequency", colormap='jet', normFactor=None): """ @@ -104,9 +104,6 @@ class SpectraPlot(Figure): zmin : None, zmax : None """ - - colormap = kwargs.get('colormap','jet') - if realtime: if not(isRealtime(utcdatatime = dataOut.utctime)): print 'Skipping this plot function' @@ -121,8 +118,10 @@ class SpectraPlot(Figure): raise ValueError, "Channel %d is not in dataOut.channelList" %channel channelIndexList.append(dataOut.channelList.index(channel)) - factor = dataOut.normFactor - + if normFactor is None: + factor = dataOut.normFactor + else: + factor = normFactor if xaxis == "frequency": x = dataOut.getFreqRange(1)/1000. xlabel = "Frequency (kHz)" @@ -283,7 +282,7 @@ class CrossSpectraPlot(Figure): save=False, figpath='./', figfile=None, ftp=False, wr_period=1, power_cmap='jet', coherence_cmap='jet', phase_cmap='RdBu_r', show=True, server=None, folder=None, username=None, password=None, - ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, + ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None, xaxis='frequency'): """ @@ -316,8 +315,11 @@ class CrossSpectraPlot(Figure): if len(pairsIndexList) > 4: pairsIndexList = pairsIndexList[0:4] - - factor = dataOut.normFactor + + if normFactor is None: + factor = dataOut.normFactor + else: + factor = normFactor x = dataOut.getVelRange(1) y = dataOut.getHeiRange() z = dataOut.data_spc[:,:,:]/factor @@ -518,10 +520,10 @@ class RTIPlot(Figure): def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True', xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None, - timerange=None, + timerange=None, colormap='jet', save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True, server=None, folder=None, username=None, password=None, - ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, **kwargs): + ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None, HEIGHT=None): """ @@ -539,7 +541,10 @@ class RTIPlot(Figure): zmax : None """ - colormap = kwargs.get('colormap', 'jet') + #colormap = kwargs.get('colormap', 'jet') + if HEIGHT is not None: + self.HEIGHT = HEIGHT + if not isTimeInHourRange(dataOut.datatime, xmin, xmax): return @@ -552,20 +557,21 @@ class RTIPlot(Figure): raise ValueError, "Channel %d is not in dataOut.channelList" channelIndexList.append(dataOut.channelList.index(channel)) - if hasattr(dataOut, 'normFactor'): + if normFactor is None: factor = dataOut.normFactor else: - factor = 1 + factor = normFactor # factor = dataOut.normFactor x = dataOut.getTimeRange() y = dataOut.getHeiRange() -# z = dataOut.data_spc/factor -# z = numpy.where(numpy.isfinite(z), z, numpy.NAN) -# avg = numpy.average(z, axis=1) -# avgdB = 10.*numpy.log10(avg) - avgdB = dataOut.getPower() + z = dataOut.data_spc/factor + z = numpy.where(numpy.isfinite(z), z, numpy.NAN) + avg = numpy.average(z, axis=1) + avgdB = 10.*numpy.log10(avg) + # avgdB = dataOut.getPower() + thisDatetime = dataOut.datatime # thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0]) @@ -1112,6 +1118,7 @@ class Noise(Figure): PREFIX = 'noise' + def __init__(self, **kwargs): Figure.__init__(self, **kwargs) self.timerange = 24*60*60 diff --git a/schainpy/model/io/MIRAtest.py b/schainpy/model/io/MIRAtest.py new file mode 100644 index 0000000..c48bc8b --- /dev/null +++ b/schainpy/model/io/MIRAtest.py @@ -0,0 +1,321 @@ +import os, sys +import glob +import fnmatch +import datetime +import time +import re +import h5py +import numpy +import matplotlib.pyplot as plt + +import pylab as plb +from scipy.optimize import curve_fit +from scipy import asarray as ar,exp +from scipy import stats + +from duplicity.path import Path +from numpy.ma.core import getdata + +SPEED_OF_LIGHT = 299792458 +SPEED_OF_LIGHT = 3e8 + +try: + from gevent import sleep +except: + from time import sleep + +from schainpy.model.data.jrodata import Spectra +#from schainpy.model.data.BLTRheaderIO import FileHeader, RecordHeader +from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation +#from schainpy.model.io.jroIO_bltr import BLTRReader +from numpy import imag, shape, NaN + + +startFp = open('/home/erick/Documents/MIRA35C/20160117/20160117_0000.zspc',"rb") + + +FILE_HEADER = numpy.dtype([ #HEADER 1024bytes + ('Hname',numpy.str_,32), #Original file name + ('Htime',numpy.str_,32), #Date and time when the file was created + ('Hoper',numpy.str_,64), #Name of operator who created the file + ('Hplace',numpy.str_,128), #Place where the measurements was carried out + ('Hdescr',numpy.str_,256), #Description of measurements + ('Hdummy',numpy.str_,512), #Reserved space + #Main chunk + ('Msign','=5 + ('SPARrawGate2',' dateFile) or (endDate < dateFile): + continue + + self.fileList.append(thisFile) + self.dateFileList.append(dateFile) + + return + + def setNextFile(self): + + file_id = self.fileIndex + + if file_id == len(self.fileList): + print '\nNo more files in the folder' + print 'Total number of file(s) read : {}'.format(self.fileIndex + 1) + self.flagNoMoreFiles = 1 + return 0 + + print '\n[Setting file] (%s) ...' % self.fileList[file_id] + filename = os.path.join(self.path, self.fileList[file_id]) + + dirname, name = os.path.split(filename) + self.siteFile = name.split('.')[0] # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya + if self.filename is not None: + self.fp.close() + self.filename = filename + self.fp = open(self.filename, 'rb') + self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1) + self.nrecords = self.header_file['nrec'][0] + self.sizeOfFile = os.path.getsize(self.filename) + self.counter_records = 0 + self.flagIsNewFile = 0 + self.fileIndex += 1 + + return 1 + + def readNextBlock(self): + + while True: + if self.counter_records == self.nrecords: + self.flagIsNewFile = 1 + if not self.setNextFile(): + return 0 + + self.readBlock() + + if (self.datatime.time() < self.startTime) or (self.datatime.time() > self.endTime): + print "[Reading] Record No. %d/%d -> %s [Skipping]" %( + self.counter_records, + self.nrecords, + self.datatime.ctime()) + continue + break + + print "[Reading] Record No. %d/%d -> %s" %( + self.counter_records, + self.nrecords, + self.datatime.ctime()) + + return 1 + + def readBlock(self): + + pointer = self.fp.tell() + header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1) + self.nchannels = header_rec['nchan'][0]/2 + self.kchan = header_rec['nrxs'][0] + self.nmodes = header_rec['nmodes'][0] + self.nranges = header_rec['nranges'][0] + self.fp.seek(pointer) + self.height = numpy.empty((self.nmodes, self.nranges)) + self.snr = numpy.empty((self.nmodes, self.nchannels, self.nranges)) + self.buffer = numpy.empty((self.nmodes, 3, self.nranges)) + + for mode in range(self.nmodes): + self.readHeader() + data = self.readData() + self.height[mode] = (data[0] - self.correction) / 1000. + self.buffer[mode] = data[1] + self.snr[mode] = data[2] + + self.counter_records = self.counter_records + self.nmodes + + return + + def readHeader(self): + ''' + RecordHeader of BLTR rawdata file + ''' + + header_structure = numpy.dtype( + REC_HEADER_STRUCTURE.descr + [ + ('antenna_coord', 'f4', (2, self.nchannels)), + ('rx_gains', 'u4', (self.nchannels,)), + ('rx_analysis', 'u4', (self.nchannels,)) + ] + ) + + self.header_rec = numpy.fromfile(self.fp, header_structure, 1) + self.lat = self.header_rec['lat'][0] + self.lon = self.header_rec['lon'][0] + self.delta = self.header_rec['delta_r'][0] + self.correction = self.header_rec['dmode_rngcorr'][0] + self.imode = self.header_rec['dmode_index'][0] + self.antenna = self.header_rec['antenna_coord'] + self.rx_gains = self.header_rec['rx_gains'] + self.time = self.header_rec['time'][0] + tseconds = self.header_rec['time'][0] + local_t1 = time.localtime(tseconds) + self.year = local_t1.tm_year + self.month = local_t1.tm_mon + self.day = local_t1.tm_mday + self.t = datetime.datetime(self.year, self.month, self.day) + self.datatime = datetime.datetime.utcfromtimestamp(self.time) + + def readData(self): + ''' + Reading and filtering data block record of BLTR rawdata file, filtering is according to status_value. + + Input: + status_value - Array data is set to NAN for values that are not equal to status_value + + ''' + + data_structure = numpy.dtype( + DATA_STRUCTURE.descr + [ + ('rx_saturation', 'u4', (self.nchannels,)), + ('chan_offset', 'u4', (2 * self.nchannels,)), + ('rx_amp', 'u4', (self.nchannels,)), + ('rx_snr', 'f4', (self.nchannels,)), + ('cross_snr', 'f4', (self.kchan,)), + ('sea_power_relative', 'f4', (self.kchan,))] + ) + + data = numpy.fromfile(self.fp, data_structure, self.nranges) + + height = data['range'] + winds = numpy.array((data['zonal'], data['meridional'], data['vertical'])) + snr = data['rx_snr'].T + + winds[numpy.where(winds == -9999.)] = numpy.nan + winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan + snr[numpy.where(snr == -9999.)] = numpy.nan + snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan + snr = numpy.power(10, snr / 10) + + return height, winds, snr + + def set_output(self): + ''' + Storing data from databuffer to dataOut object + ''' + + self.dataOut.data_SNR = self.snr + self.dataOut.height = self.height + self.dataOut.data_output = self.buffer + self.dataOut.utctimeInit = self.time + self.dataOut.utctime = self.dataOut.utctimeInit + self.dataOut.useLocalTime = False + self.dataOut.paramInterval = 157 + self.dataOut.timezone = self.timezone + self.dataOut.site = self.siteFile + self.dataOut.nrecords = self.nrecords/self.nmodes + self.dataOut.sizeOfFile = self.sizeOfFile + self.dataOut.lat = self.lat + self.dataOut.lon = self.lon + self.dataOut.channelList = range(self.nchannels) + self.dataOut.kchan = self.kchan + # self.dataOut.nHeights = self.nranges + self.dataOut.delta = self.delta + self.dataOut.correction = self.correction + self.dataOut.nmodes = self.nmodes + self.dataOut.imode = self.imode + self.dataOut.antenna = self.antenna + self.dataOut.rx_gains = self.rx_gains + self.dataOut.flagNoData = False + + def getData(self): + ''' + Storing data from databuffer to dataOut object + ''' + if self.flagNoMoreFiles: + self.dataOut.flagNoData = True + print 'No file left to process' + return 0 + + if not self.readNextBlock(): + self.dataOut.flagNoData = True + return 0 + + self.set_output() + + return 1 diff --git a/schainpy/model/io/jroIO_base.py b/schainpy/model/io/jroIO_base.py index 0b2e559..668e1e8 100644 --- a/schainpy/model/io/jroIO_base.py +++ b/schainpy/model/io/jroIO_base.py @@ -10,7 +10,8 @@ import time import numpy import fnmatch import inspect -import time, datetime +import time +import datetime import traceback import zmq @@ -24,6 +25,7 @@ from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, ge LOCALTIME = True + def isNumber(cad): """ Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero. @@ -38,11 +40,12 @@ def isNumber(cad): False : no es un string numerico """ try: - float( cad ) + float(cad) return True except: return False + def isFileInEpoch(filename, startUTSeconds, endUTSeconds): """ Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado. @@ -67,16 +70,16 @@ def isFileInEpoch(filename, startUTSeconds, endUTSeconds): basicHeaderObj = BasicHeader(LOCALTIME) try: - fp = open(filename,'rb') + fp = open(filename, 'rb') except IOError: - print "The file %s can't be opened" %(filename) + print "The file %s can't be opened" % (filename) return 0 sts = basicHeaderObj.read(fp) fp.close() if not(sts): - print "Skipping the file %s because it has not a valid header" %(filename) + print "Skipping the file %s because it has not a valid header" % (filename) return 0 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)): @@ -84,6 +87,7 @@ def isFileInEpoch(filename, startUTSeconds, endUTSeconds): return 1 + def isTimeInRange(thisTime, startTime, endTime): if endTime >= startTime: @@ -97,6 +101,7 @@ def isTimeInRange(thisTime, startTime, endTime): return 1 + def isFileInTimeRange(filename, startDate, endDate, startTime, endTime): """ Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. @@ -122,11 +127,10 @@ def isFileInTimeRange(filename, startDate, endDate, startTime, endTime): """ - try: - fp = open(filename,'rb') + fp = open(filename, 'rb') except IOError: - print "The file %s can't be opened" %(filename) + print "The file %s can't be opened" % (filename) return None firstBasicHeaderObj = BasicHeader(LOCALTIME) @@ -139,7 +143,7 @@ def isFileInTimeRange(filename, startDate, endDate, startTime, endTime): sts = firstBasicHeaderObj.read(fp) if not(sts): - print "[Reading] Skipping the file %s because it has not a valid header" %(filename) + print "[Reading] Skipping the file %s because it has not a valid header" % (filename) return None if not systemHeaderObj.read(fp): @@ -153,10 +157,10 @@ def isFileInTimeRange(filename, startDate, endDate, startTime, endTime): filesize = os.path.getsize(filename) - offset = processingHeaderObj.blockSize + 24 #header size + offset = processingHeaderObj.blockSize + 24 # header size if filesize <= offset: - print "[Reading] %s: This file has not enough data" %filename + print "[Reading] %s: This file has not enough data" % filename return None fp.seek(-offset, 2) @@ -172,7 +176,7 @@ def isFileInTimeRange(filename, startDate, endDate, startTime, endTime): thisDate = thisDatetime.date() thisTime_first_block = thisDatetime.time() - #General case + # General case # o>>>>>>>>>>>>>><<<<<<<<<<<<<>>>>>>>>>> #-----------o----------------------------o----------- @@ -201,6 +204,7 @@ def isFileInTimeRange(filename, startDate, endDate, startTime, endTime): return thisDatetime + def isFolderInDateRange(folder, startDate=None, endDate=None): """ Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. @@ -227,7 +231,7 @@ def isFolderInDateRange(folder, startDate=None, endDate=None): basename = os.path.basename(folder) if not isRadarFolder(basename): - print "The folder %s has not the rigth format" %folder + print "The folder %s has not the rigth format" % folder return 0 if startDate and endDate: @@ -241,6 +245,7 @@ def isFolderInDateRange(folder, startDate=None, endDate=None): return 1 + def isFileInDateRange(filename, startDate=None, endDate=None): """ Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. @@ -269,7 +274,7 @@ def isFileInDateRange(filename, startDate=None, endDate=None): basename = os.path.basename(filename) if not isRadarFile(basename): - print "The filename %s has not the rigth format" %filename + print "The filename %s has not the rigth format" % filename return 0 if startDate and endDate: @@ -283,6 +288,7 @@ def isFileInDateRange(filename, startDate=None, endDate=None): return 1 + def getFileFromSet(path, ext, set): validFilelist = [] fileList = os.listdir(path) @@ -293,7 +299,7 @@ def getFileFromSet(path, ext, set): for thisFile in fileList: try: year = int(thisFile[1:5]) - doy = int(thisFile[5:8]) + doy = int(thisFile[5:8]) except: continue @@ -302,21 +308,23 @@ def getFileFromSet(path, ext, set): validFilelist.append(thisFile) - myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set)) + myfile = fnmatch.filter( + validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set)) - if len(myfile)!= 0: + if len(myfile) != 0: return myfile[0] else: - filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower()) - print 'the filename %s does not exist'%filename + filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower()) + print 'the filename %s does not exist' % filename print '...going to the last file: ' if validFilelist: - validFilelist = sorted( validFilelist, key=str.lower ) + validFilelist = sorted(validFilelist, key=str.lower) return validFilelist[-1] return None + def getlastFileFromPath(path, ext): """ Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext" @@ -354,11 +362,12 @@ def getlastFileFromPath(path, ext): validFilelist.append(thisFile) if validFilelist: - validFilelist = sorted( validFilelist, key=str.lower ) + validFilelist = sorted(validFilelist, key=str.lower) return validFilelist[-1] return None + def checkForRealPath(path, foldercounter, year, doy, set, ext): """ Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path, @@ -386,28 +395,32 @@ def checkForRealPath(path, foldercounter, year, doy, set, ext): find_flag = False filename = None - prefixDirList = [None,'d','D'] - if ext.lower() == ".r": #voltage - prefixFileList = ['d','D'] - elif ext.lower() == ".pdata": #spectra - prefixFileList = ['p','P'] + prefixDirList = [None, 'd', 'D'] + if ext.lower() == ".r": # voltage + prefixFileList = ['d', 'D'] + elif ext.lower() == ".pdata": # spectra + prefixFileList = ['p', 'P'] else: return None, filename - #barrido por las combinaciones posibles + # barrido por las combinaciones posibles for prefixDir in prefixDirList: thispath = path if prefixDir != None: - #formo el nombre del directorio xYYYYDDD (x=d o x=D) + # formo el nombre del directorio xYYYYDDD (x=d o x=D) if foldercounter == 0: - thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy )) + thispath = os.path.join(path, "%s%04d%03d" % + (prefixDir, year, doy)) else: - thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter)) - for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D" - filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext - fullfilename = os.path.join( thispath, filename ) #formo el path completo - - if os.path.exists( fullfilename ): #verifico que exista + thispath = os.path.join(path, "%s%04d%03d_%02d" % ( + prefixDir, year, doy, foldercounter)) + for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D" + # formo el nombre del file xYYYYDDDSSS.ext + filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext) + fullfilename = os.path.join( + thispath, filename) # formo el path completo + + if os.path.exists(fullfilename): # verifico que exista find_flag = True break if find_flag: @@ -418,6 +431,7 @@ def checkForRealPath(path, foldercounter, year, doy, set, ext): return fullfilename, filename + def isRadarFolder(folder): try: year = int(folder[1:5]) @@ -427,15 +441,17 @@ def isRadarFolder(folder): return 1 + def isRadarFile(file): - try: - year = int(file[1:5]) - doy = int(file[5:8]) - set = int(file[8:11]) - except: - return 0 + try: + year = int(file[1:5]) + doy = int(file[5:8]) + set = int(file[8:11]) + except: + return 0 + + return 1 - return 1 def getDateFromRadarFile(file): try: @@ -445,9 +461,10 @@ def getDateFromRadarFile(file): except: return None - thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1) + thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1) return thisDate + def getDateFromRadarFolder(folder): try: year = int(folder[1:5]) @@ -455,9 +472,10 @@ def getDateFromRadarFolder(folder): except: return None - thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1) + thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1) return thisDate + class JRODataIO: c = 3E8 @@ -540,6 +558,7 @@ class JRODataIO: def getAllowedArgs(self): return inspect.getargspec(self.run).args + class JRODataReader(JRODataIO): online = 0 @@ -548,11 +567,11 @@ class JRODataReader(JRODataIO): nReadBlocks = 0 - delay = 10 #number of seconds waiting a new file + delay = 10 # number of seconds waiting a new file - nTries = 3 #quantity tries + nTries = 3 # quantity tries - nFiles = 3 #number of files for searching + nFiles = 3 # number of files for searching path = None @@ -572,14 +591,13 @@ class JRODataReader(JRODataIO): txIndex = None - #Added-------------------- + # Added-------------------- selBlocksize = None selBlocktime = None def __init__(self): - """ This class is used to find data files @@ -590,7 +608,6 @@ class JRODataReader(JRODataIO): """ pass - def createObjByDefault(self): """ @@ -605,8 +622,8 @@ class JRODataReader(JRODataIO): path, startDate=None, endDate=None, - startTime=datetime.time(0,0,0), - endTime=datetime.time(23,59,59), + startTime=datetime.time(0, 0, 0), + endTime=datetime.time(23, 59, 59), set=None, expLabel='', ext='.r', @@ -619,22 +636,23 @@ class JRODataReader(JRODataIO): pathList = [] - dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True) + dateList, pathList = self.findDatafiles( + path, startDate, endDate, expLabel, ext, walk, include_path=True) if dateList == []: return [], [] if len(dateList) > 1: - print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList)) + print "[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)) else: - print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0]) + print "[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]) filenameList = [] datetimeList = [] for thisPath in pathList: - fileList = glob.glob1(thisPath, "*%s" %ext) + fileList = glob.glob1(thisPath, "*%s" % ext) fileList.sort() skippedFileList = [] @@ -644,19 +662,21 @@ class JRODataReader(JRODataIO): if skip == 0: skippedFileList = [] else: - skippedFileList = fileList[cursor*skip: cursor*skip + skip] + skippedFileList = fileList[cursor * + skip: cursor * skip + skip] else: skippedFileList = fileList for file in skippedFileList: - filename = os.path.join(thisPath,file) + filename = os.path.join(thisPath, file) if not isFileInDateRange(filename, startDate, endDate): continue - thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime) + thisDatetime = isFileInTimeRange( + filename, startDate, endDate, startTime, endTime) if not(thisDatetime): continue @@ -665,10 +685,10 @@ class JRODataReader(JRODataIO): datetimeList.append(thisDatetime) if not(filenameList): - print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path) + print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path) return [], [] - print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime) + print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime) print # for i in range(len(filenameList)): @@ -679,8 +699,7 @@ class JRODataReader(JRODataIO): return pathList, filenameList - def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None): - + def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None): """ Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y devuelve el archivo encontrado ademas de otros datos. @@ -712,9 +731,9 @@ class JRODataReader(JRODataIO): fullpath = path foldercounter = 0 else: - #Filtra solo los directorios + # Filtra solo los directorios for thisPath in os.listdir(path): - if not os.path.isdir(os.path.join(path,thisPath)): + if not os.path.isdir(os.path.join(path, thisPath)): continue if not isRadarFolder(thisPath): continue @@ -724,14 +743,14 @@ class JRODataReader(JRODataIO): if not(dirList): return None, None, None, None, None, None - dirList = sorted( dirList, key=str.lower ) + dirList = sorted(dirList, key=str.lower) doypath = dirList[-1] - foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0 + foldercounter = int(doypath.split('_')[1]) if len( + doypath.split('_')) > 1 else 0 fullpath = os.path.join(path, doypath, expLabel) - - print "[Reading] %s folder was found: " %(fullpath ) + print "[Reading] %s folder was found: " % (fullpath) if set == None: filename = getlastFileFromPath(fullpath, ext) @@ -741,14 +760,14 @@ class JRODataReader(JRODataIO): if not(filename): return None, None, None, None, None, None - print "[Reading] %s file was found" %(filename) + print "[Reading] %s file was found" % (filename) if not(self.__verifyFile(os.path.join(fullpath, filename))): return None, None, None, None, None, None - year = int( filename[1:5] ) - doy = int( filename[5:8] ) - set = int( filename[8:11] ) + year = int(filename[1:5]) + doy = int(filename[5:8]) + set = int(filename[8:11]) return fullpath, foldercounter, filename, year, doy, set @@ -769,7 +788,7 @@ class JRODataReader(JRODataIO): continue fileSize = os.path.getsize(filename) - fp = open(filename,'rb') + fp = open(filename, 'rb') break self.flagIsNewFile = 1 @@ -813,29 +832,32 @@ class JRODataReader(JRODataIO): self.set = 0 self.foldercounter += 1 - #busca el 1er file disponible - fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext ) + # busca el 1er file disponible + fullfilename, filename = checkForRealPath( + self.path, self.foldercounter, self.year, self.doy, self.set, self.ext) if fullfilename: if self.__verifyFile(fullfilename, False): fileOk_flag = True - #si no encuentra un file entonces espera y vuelve a buscar + # si no encuentra un file entonces espera y vuelve a buscar if not(fileOk_flag): - for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles + # busco en los siguientes self.nFiles+1 files posibles + for nFiles in range(self.nFiles + 1): - if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces + if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces tries = self.nTries else: - tries = 1 #si no es la 1era vez entonces solo lo hace una vez + tries = 1 # si no es la 1era vez entonces solo lo hace una vez - for nTries in range( tries ): + for nTries in range(tries): if firstTime_flag: - print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 ) - sleep( self.delay ) + print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1) + sleep(self.delay) else: print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext) - fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext ) + fullfilename, filename = checkForRealPath( + self.path, self.foldercounter, self.year, self.doy, self.set, self.ext) if fullfilename: if self.__verifyFile(fullfilename): fileOk_flag = True @@ -849,16 +871,18 @@ class JRODataReader(JRODataIO): print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename self.set += 1 - if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta + # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta + if nFiles == (self.nFiles - 1): self.set = 0 self.doy += 1 self.foldercounter = 0 if fileOk_flag: - self.fileSize = os.path.getsize( fullfilename ) + self.fileSize = os.path.getsize(fullfilename) self.filename = fullfilename self.flagIsNewFile = 1 - if self.fp != None: self.fp.close() + if self.fp != None: + self.fp.close() self.fp = open(fullfilename, 'rb') self.flagNoMoreFiles = 0 # print '[Reading] Setting the file: %s' % fullfilename @@ -908,48 +932,47 @@ class JRODataReader(JRODataIO): neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize - for nTries in range( self.nTries ): + for nTries in range(self.nTries): self.fp.close() - self.fp = open( self.filename, 'rb' ) - self.fp.seek( currentPointer ) + self.fp = open(self.filename, 'rb') + self.fp.seek(currentPointer) - self.fileSize = os.path.getsize( self.filename ) + self.fileSize = os.path.getsize(self.filename) currentSize = self.fileSize - currentPointer - if ( currentSize >= neededSize ): + if (currentSize >= neededSize): self.basicHeaderObj.read(self.fp) return 1 if self.fileSize == self.fileSizeByHeader: -# self.flagEoF = True + # self.flagEoF = True return 0 - print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1) - sleep( self.delay ) - + print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1) + sleep(self.delay) return 0 - def waitDataBlock(self,pointer_location): + def waitDataBlock(self, pointer_location): currentPointer = pointer_location - neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize + neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize - for nTries in range( self.nTries ): + for nTries in range(self.nTries): self.fp.close() - self.fp = open( self.filename, 'rb' ) - self.fp.seek( currentPointer ) + self.fp = open(self.filename, 'rb') + self.fp.seek(currentPointer) - self.fileSize = os.path.getsize( self.filename ) + self.fileSize = os.path.getsize(self.filename) currentSize = self.fileSize - currentPointer - if ( currentSize >= neededSize ): + if (currentSize >= neededSize): return 1 - print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1) - sleep( self.delay ) + print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1) + sleep(self.delay) return 0 @@ -961,7 +984,7 @@ class JRODataReader(JRODataIO): csize = self.fileSize - self.fp.tell() blocksize = self.processingHeaderObj.blockSize - #salta el primer bloque de datos + # salta el primer bloque de datos if csize > self.processingHeaderObj.blockSize: self.fp.seek(self.fp.tell() + blocksize) else: @@ -971,7 +994,7 @@ class JRODataReader(JRODataIO): neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize while True: - if self.fp.tell()= neededSize): @@ -1018,11 +1041,11 @@ class JRODataReader(JRODataIO): if self.__waitNewBlock(): self.lastUTTime = self.basicHeaderObj.utc return 1 - #if self.server is None: + # if self.server is None: if not(self.setNextFile()): return 0 - deltaTime = self.basicHeaderObj.utc - self.lastUTTime # + deltaTime = self.basicHeaderObj.utc - self.lastUTTime self.lastUTTime = self.basicHeaderObj.utc self.flagDiscontinuousBlock = 0 @@ -1034,11 +1057,11 @@ class JRODataReader(JRODataIO): def readNextBlock(self): - #Skip block out of startTime and endTime - while True: - if not(self.__setNewBlock()): + # Skip block out of startTime and endTime + while True: + if not(self.__setNewBlock()): return 0 - + if not(self.readBlock()): return 0 @@ -1046,17 +1069,17 @@ class JRODataReader(JRODataIO): if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime): - print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks, - self.processingHeaderObj.dataBlocksPerFile, - self.dataOut.datatime.ctime()) + print "[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks, + self.processingHeaderObj.dataBlocksPerFile, + self.dataOut.datatime.ctime()) continue break if self.verbose: - print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks, - self.processingHeaderObj.dataBlocksPerFile, - self.dataOut.datatime.ctime()) + print "[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks, + self.processingHeaderObj.dataBlocksPerFile, + self.dataOut.datatime.ctime()) return 1 def __readFirstHeader(self): @@ -1068,25 +1091,28 @@ class JRODataReader(JRODataIO): self.firstHeaderSize = self.basicHeaderObj.size - datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR)) + datatype = int(numpy.log2((self.processingHeaderObj.processFlags & + PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR)) if datatype == 0: - datatype_str = numpy.dtype([('real',' %s" %(self.nReadBlocks, - self.processingHeaderObj.dataBlocksPerFile, - self.dataOut.datatime.ctime()) +# if self.flagIsNewBlock: +# print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks, +# self.processingHeaderObj.dataBlocksPerFile, +# self.dataOut.datatime.ctime()) def printInfo(self): @@ -1440,25 +1472,29 @@ class JRODataReader(JRODataIO): path=None, startDate=None, endDate=None, - startTime=datetime.time(0,0,0), - endTime=datetime.time(23,59,59), + startTime=datetime.time(0, 0, 0), + endTime=datetime.time(23, 59, 59), set=None, - expLabel = "", - ext = None, - online = False, - delay = 60, - walk = True, - getblock = False, - nTxs = 1, + expLabel="", + ext=None, + online=False, + delay=60, + walk=True, + getblock=False, + nTxs=1, realtime=False, blocksize=None, blocktime=None, - queue=None, skip=None, cursor=None, warnings=True, server=None, - verbose=True, **kwargs): + verbose=True, + format=None, + oneDDict=None, + twoDDict=None, + ind2DList=None, **kwargs): + if not(self.isConfig): self.setup(path=path, startDate=startDate, @@ -1480,13 +1516,18 @@ class JRODataReader(JRODataIO): cursor=cursor, warnings=warnings, server=server, - verbose=verbose) + verbose=verbose, + format=format, + oneDDict=oneDDict, + twoDDict=twoDDict, + ind2DList=ind2DList) self.isConfig = True if server is None: self.getData() - else: + else: self.getFromServer() + class JRODataWriter(JRODataIO): """ @@ -1511,23 +1552,18 @@ class JRODataWriter(JRODataIO): def __init__(self, dataOut=None): raise NotImplementedError - def hasAllDataInBuffer(self): raise NotImplementedError - def setBlockDimension(self): raise NotImplementedError - def writeBlock(self): raise NotImplementedError - def putData(self): raise NotImplementedError - def getProcessFlags(self): processFlags = 0 @@ -1563,12 +1599,12 @@ class JRODataWriter(JRODataIO): def setBasicHeader(self): - self.basicHeaderObj.size = self.basicHeaderSize #bytes + self.basicHeaderObj.size = self.basicHeaderSize # bytes self.basicHeaderObj.version = self.versionFile self.basicHeaderObj.dataBlock = self.nTotalBlocks utc = numpy.floor(self.dataOut.utctime) - milisecond = (self.dataOut.utctime - utc)* 1000.0 + milisecond = (self.dataOut.utctime - utc) * 1000.0 self.basicHeaderObj.utc = utc self.basicHeaderObj.miliSecond = milisecond @@ -1606,7 +1642,8 @@ class JRODataWriter(JRODataIO): # CALCULAR PARAMETROS - sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size + sizeLongHeader = self.systemHeaderObj.size + \ + self.radarControllerHeaderObj.size + self.processingHeaderObj.size self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader self.basicHeaderObj.write(self.fp) @@ -1632,12 +1669,11 @@ class JRODataWriter(JRODataIO): self.basicHeaderObj.write(self.fp) return 1 - if not( self.setNextFile() ): + if not(self.setNextFile()): return 0 return 1 - def writeNextBlock(self): """ Selecciona el bloque siguiente de datos y los escribe en un file @@ -1646,13 +1682,13 @@ class JRODataWriter(JRODataIO): 0 : Si no hizo pudo escribir el bloque de datos 1 : Si no pudo escribir el bloque de datos """ - if not( self.__setNewBlock() ): + if not(self.__setNewBlock()): return 0 self.writeBlock() - print "[Writing] Block No. %d/%d" %(self.blockIndex, - self.processingHeaderObj.dataBlocksPerFile) + print "[Writing] Block No. %d/%d" % (self.blockIndex, + self.processingHeaderObj.dataBlocksPerFile) return 1 @@ -1677,46 +1713,48 @@ class JRODataWriter(JRODataIO): if self.fp != None: self.fp.close() - timeTuple = time.localtime( self.dataOut.utctime) - subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) + timeTuple = time.localtime(self.dataOut.utctime) + subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday) - fullpath = os.path.join( path, subfolder ) + fullpath = os.path.join(path, subfolder) setFile = self.setFile - if not( os.path.exists(fullpath) ): + if not(os.path.exists(fullpath)): os.mkdir(fullpath) - setFile = -1 #inicializo mi contador de seteo + setFile = -1 # inicializo mi contador de seteo else: - filesList = os.listdir( fullpath ) - if len( filesList ) > 0: - filesList = sorted( filesList, key=str.lower ) + filesList = os.listdir(fullpath) + if len(filesList) > 0: + filesList = sorted(filesList, key=str.lower) filen = filesList[-1] # el filename debera tener el siguiente formato # 0 1234 567 89A BCDE (hex) # x YYYY DDD SSS .ext - if isNumber( filen[8:11] ): - setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file + if isNumber(filen[8:11]): + # inicializo mi contador de seteo al seteo del ultimo file + setFile = int(filen[8:11]) else: setFile = -1 else: - setFile = -1 #inicializo mi contador de seteo + setFile = -1 # inicializo mi contador de seteo setFile += 1 - #If this is a new day it resets some values + # If this is a new day it resets some values if self.dataOut.datatime.date() > self.fileDate: setFile = 0 self.nTotalBlocks = 0 - filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext ) + filen = '%s%4.4d%3.3d%3.3d%s' % ( + self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext) - filename = os.path.join( path, subfolder, filen ) + filename = os.path.join(path, subfolder, filen) - fp = open( filename,'wb' ) + fp = open(filename, 'wb') self.blockIndex = 0 - #guardando atributos + # guardando atributos self.filename = filename self.subfolder = subfolder self.fp = fp @@ -1726,7 +1764,7 @@ class JRODataWriter(JRODataIO): self.setFirstHeader() - print '[Writing] Opening file: %s'%self.filename + print '[Writing] Opening file: %s' % self.filename self.__writeFirstHeader() @@ -1771,7 +1809,7 @@ class JRODataWriter(JRODataIO): self.dataOut = dataOut self.fileDate = self.dataOut.datatime.date() - #By default + # By default self.dtype = self.dataOut.dtype if datatype is not None: @@ -1789,7 +1827,8 @@ class JRODataWriter(JRODataIO): if not(self.isConfig): - self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs) + self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, + set=set, ext=ext, datatype=datatype, **kwargs) self.isConfig = True self.putData() diff --git a/schainpy/model/io/jroIO_bltr.py b/schainpy/model/io/jroIO_bltr.py new file mode 100644 index 0000000..fe46699 --- /dev/null +++ b/schainpy/model/io/jroIO_bltr.py @@ -0,0 +1,1154 @@ +import os, sys +import glob +import fnmatch +import datetime +import time +import re +import h5py +import numpy +import matplotlib.pyplot as plt + +import pylab as plb +from scipy.optimize import curve_fit +from scipy import asarray as ar, exp +from scipy import stats + +from numpy.ma.core import getdata + +SPEED_OF_LIGHT = 299792458 +SPEED_OF_LIGHT = 3e8 + +try: + from gevent import sleep +except: + from time import sleep + +from schainpy.model.data.jrodata import Spectra +#from schainpy.model.data.BLTRheaderIO import FileHeader, RecordHeader +from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation +#from schainpy.model.io.jroIO_bltr import BLTRReader +from numpy import imag, shape, NaN + +from jroIO_base import JRODataReader + + +class Header(object): + + def __init__(self): + raise NotImplementedError + + + def read(self): + + raise NotImplementedError + + def write(self): + + raise NotImplementedError + + def printInfo(self): + + message = "#"*50 + "\n" + message += self.__class__.__name__.upper() + "\n" + message += "#"*50 + "\n" + + keyList = self.__dict__.keys() + keyList.sort() + + for key in keyList: + message += "%s = %s" %(key, self.__dict__[key]) + "\n" + + if "size" not in keyList: + attr = getattr(self, "size") + + if attr: + message += "%s = %s" %("size", attr) + "\n" + + #print message + + + + + +FILE_STRUCTURE = numpy.dtype([ #HEADER 48bytes + ('FileMgcNumber',' vertical) + ('AntennaCoord0',' endFp: + sys.stderr.write("Warning %s: Size value read from System Header is lower than it has to be\n" %fp) + return 0 + + if OffRHeader < endFp: + sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" %fp) + return 0 + + return 1 + + +class BLTRSpectraReader (ProcessingUnit, FileHeaderBLTR, RecordHeaderBLTR, JRODataReader): + + path = None + startDate = None + endDate = None + startTime = None + endTime = None + walk = None + isConfig = False + + + fileList= None + + #metadata + TimeZone= None + Interval= None + heightList= None + + #data + data= None + utctime= None + + + + def __init__(self, **kwargs): + + #Eliminar de la base la herencia + ProcessingUnit.__init__(self, **kwargs) + + #self.isConfig = False + + #self.pts2read_SelfSpectra = 0 + #self.pts2read_CrossSpectra = 0 + #self.pts2read_DCchannels = 0 + #self.datablock = None + self.utc = None + self.ext = ".fdt" + self.optchar = "P" + self.fpFile=None + self.fp = None + self.BlockCounter=0 + self.dtype = None + self.fileSizeByHeader = None + self.filenameList = [] + self.fileSelector = 0 + self.Off2StartNxtRec=0 + self.RecCounter=0 + self.flagNoMoreFiles = 0 + self.data_spc=None + self.data_cspc=None + self.data_output=None + self.path = None + self.OffsetStartHeader=0 + self.Off2StartData=0 + self.ipp = 0 + self.nFDTdataRecors=0 + self.blocksize = 0 + self.dataOut = Spectra() + self.profileIndex = 1 #Always + self.dataOut.flagNoData=False + self.dataOut.nRdPairs = 0 + self.dataOut.pairsList = [] + self.dataOut.data_spc=None + self.dataOut.noise=[] + self.dataOut.velocityX=[] + self.dataOut.velocityY=[] + self.dataOut.velocityV=[] + + + + def Files2Read(self, fp): + ''' + Function that indicates the number of .fdt files that exist in the folder to be read. + It also creates an organized list with the names of the files to read. + ''' + #self.__checkPath() + + ListaData=os.listdir(fp) #Gets the list of files within the fp address + ListaData=sorted(ListaData) #Sort the list of files from least to largest by names + nFiles=0 #File Counter + FileList=[] #A list is created that will contain the .fdt files + for IndexFile in ListaData : + if '.fdt' in IndexFile: + FileList.append(IndexFile) + nFiles+=1 + + #print 'Files2Read' + #print 'Existen '+str(nFiles)+' archivos .fdt' + + self.filenameList=FileList #List of files from least to largest by names + + + def run(self, **kwargs): + ''' + This method will be the one that will initiate the data entry, will be called constantly. + You should first verify that your Setup () is set up and then continue to acquire + the data to be processed with getData (). + ''' + if not self.isConfig: + self.setup(**kwargs) + self.isConfig = True + + self.getData() + #print 'running' + + + def setup(self, path=None, + startDate=None, + endDate=None, + startTime=None, + endTime=None, + walk=True, + timezone='utc', + code = None, + online=False, + ReadMode=None, + **kwargs): + + self.isConfig = True + + self.path=path + self.startDate=startDate + self.endDate=endDate + self.startTime=startTime + self.endTime=endTime + self.walk=walk + self.ReadMode=int(ReadMode) + + pass + + + def getData(self): + ''' + Before starting this function, you should check that there is still an unread file, + If there are still blocks to read or if the data block is empty. + + You should call the file "read". + + ''' + + if self.flagNoMoreFiles: + self.dataOut.flagNoData = True + print 'NoData se vuelve true' + return 0 + + self.fp=self.path + self.Files2Read(self.fp) + self.readFile(self.fp) + self.dataOut.data_spc = self.data_spc + self.dataOut.data_cspc =self.data_cspc + self.dataOut.data_output=self.data_output + + print 'self.dataOut.data_output', shape(self.dataOut.data_output) + + #self.removeDC() + return self.dataOut.data_spc + + + def readFile(self,fp): + ''' + You must indicate if you are reading in Online or Offline mode and load the + The parameters for this file reading mode. + + Then you must do 2 actions: + + 1. Get the BLTR FileHeader. + 2. Start reading the first block. + ''' + + #The address of the folder is generated the name of the .fdt file that will be read + print "File: ",self.fileSelector+1 + + if self.fileSelector < len(self.filenameList): + + self.fpFile=str(fp)+'/'+str(self.filenameList[self.fileSelector]) + #print self.fpFile + fheader = FileHeaderBLTR() + fheader.FHread(self.fpFile) #Bltr FileHeader Reading + self.nFDTdataRecors=fheader.nFDTdataRecors + + self.readBlock() #Block reading + else: + print 'readFile FlagNoData becomes true' + self.flagNoMoreFiles=True + self.dataOut.flagNoData = True + return 0 + + def getVelRange(self, extrapoints=0): + Lambda= SPEED_OF_LIGHT/50000000 + PRF = self.dataOut.PRF#1./(self.dataOut.ippSeconds * self.dataOut.nCohInt) + Vmax=-Lambda/(4.*(1./PRF)*self.dataOut.nCohInt*2.) + deltafreq = PRF / (self.nProfiles) + deltavel = (Vmax*2) / (self.nProfiles) + freqrange = deltafreq*(numpy.arange(self.nProfiles)-self.nProfiles/2.) - deltafreq/2 + velrange = deltavel*(numpy.arange(self.nProfiles)-self.nProfiles/2.) + return velrange + + def readBlock(self): + ''' + It should be checked if the block has data, if it is not passed to the next file. + + Then the following is done: + + 1. Read the RecordHeader + 2. Fill the buffer with the current block number. + + ''' + + if self.BlockCounter < self.nFDTdataRecors-2: + print self.nFDTdataRecors, 'CONDICION!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' + if self.ReadMode==1: + rheader = RecordHeaderBLTR(RecCounter=self.BlockCounter+1) + elif self.ReadMode==0: + rheader = RecordHeaderBLTR(RecCounter=self.BlockCounter) + + rheader.RHread(self.fpFile) #Bltr FileHeader Reading + + self.OffsetStartHeader=rheader.OffsetStartHeader + self.RecCounter=rheader.RecCounter + self.Off2StartNxtRec=rheader.Off2StartNxtRec + self.Off2StartData=rheader.Off2StartData + self.nProfiles=rheader.nProfiles + self.nChannels=rheader.nChannels + self.nHeights=rheader.nHeights + self.frequency=rheader.TransmitFrec + self.DualModeIndex=rheader.DualModeIndex + + self.pairsList =[(0,1),(0,2),(1,2)] + self.dataOut.pairsList = self.pairsList + + self.nRdPairs=len(self.dataOut.pairsList) + self.dataOut.nRdPairs = self.nRdPairs + + self.__firstHeigth=rheader.StartRangeSamp + self.__deltaHeigth=rheader.SampResolution + self.dataOut.heightList= self.__firstHeigth + numpy.array(range(self.nHeights))*self.__deltaHeigth + self.dataOut.channelList = range(self.nChannels) + self.dataOut.nProfiles=rheader.nProfiles + self.dataOut.nIncohInt=rheader.nIncohInt + self.dataOut.nCohInt=rheader.nCohInt + self.dataOut.ippSeconds= 1/float(rheader.PRFhz) + self.dataOut.PRF=rheader.PRFhz + self.dataOut.nFFTPoints=rheader.nProfiles + self.dataOut.utctime=rheader.nUtime + self.dataOut.timeZone=0 + self.dataOut.normFactor= self.dataOut.nProfiles*self.dataOut.nIncohInt*self.dataOut.nCohInt + self.dataOut.outputInterval= self.dataOut.ippSeconds * self.dataOut.nCohInt * self.dataOut.nIncohInt * self.nProfiles + + self.data_output=numpy.ones([3,rheader.nHeights])*numpy.NaN + print 'self.data_output', shape(self.data_output) + self.dataOut.velocityX=[] + self.dataOut.velocityY=[] + self.dataOut.velocityV=[] + + '''Block Reading, the Block Data is received and Reshape is used to give it + shape. + ''' + + #Procedure to take the pointer to where the date block starts + startDATA = open(self.fpFile,"rb") + OffDATA= self.OffsetStartHeader + self.RecCounter*self.Off2StartNxtRec+self.Off2StartData + startDATA.seek(OffDATA, os.SEEK_SET) + + def moving_average(x, N=2): + return numpy.convolve(x, numpy.ones((N,))/N)[(N-1):] + + def gaus(xSamples,a,x0,sigma): + return a*exp(-(xSamples-x0)**2/(2*sigma**2)) + + def Find(x,value): + for index in range(len(x)): + if x[index]==value: + return index + + def pol2cart(rho, phi): + x = rho * numpy.cos(phi) + y = rho * numpy.sin(phi) + return(x, y) + + + + + if self.DualModeIndex==self.ReadMode: + + self.data_fft = numpy.fromfile( startDATA, [('complex',' 0.0001) : +# +# try: +# popt,pcov = curve_fit(gaus,xSamples,yMean,p0=[1,meanGauss,sigma]) +# +# if numpy.amax(popt)>numpy.amax(yMean)*0.3: +# FitGauss=gaus(xSamples,*popt) +# +# else: +# FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean) +# print 'Verificador: Dentro', Height +# except RuntimeError: +# +# try: +# for j in range(len(ySamples[1])): +# yMean2=numpy.append(yMean2,numpy.average([ySamples[1,j],ySamples[2,j]])) +# popt,pcov = curve_fit(gaus,xSamples,yMean2,p0=[1,meanGauss,sigma]) +# FitGauss=gaus(xSamples,*popt) +# print 'Verificador: Exepcion1', Height +# except RuntimeError: +# +# try: +# popt,pcov = curve_fit(gaus,xSamples,ySamples[1],p0=[1,meanGauss,sigma]) +# FitGauss=gaus(xSamples,*popt) +# print 'Verificador: Exepcion2', Height +# except RuntimeError: +# FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean) +# print 'Verificador: Exepcion3', Height +# else: +# FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean) +# #print 'Verificador: Fuera', Height +# +# +# +# Maximun=numpy.amax(yMean) +# eMinus1=Maximun*numpy.exp(-1) +# +# HWpos=Find(FitGauss,min(FitGauss, key=lambda value:abs(value-eMinus1))) +# HalfWidth= xFrec[HWpos] +# GCpos=Find(FitGauss, numpy.amax(FitGauss)) +# Vpos=Find(FactNorm, numpy.amax(FactNorm)) +# #Vpos=numpy.sum(FactNorm)/len(FactNorm) +# #Vpos=Find(FactNorm, min(FactNorm, key=lambda value:abs(value- numpy.mean(FactNorm) ))) +# #print 'GCpos',GCpos, numpy.amax(FitGauss), 'HWpos',HWpos +# '''****** Getting Fij ******''' +# +# GaussCenter=xFrec[GCpos] +# if (GaussCenter<0 and HalfWidth>0) or (GaussCenter>0 and HalfWidth<0): +# Fij=abs(GaussCenter)+abs(HalfWidth)+0.0000001 +# else: +# Fij=abs(GaussCenter-HalfWidth)+0.0000001 +# +# '''****** Getting Frecuency range of significant data ******''' +# +# Rangpos=Find(FitGauss,min(FitGauss, key=lambda value:abs(value-Maximun*0.10))) +# +# if Rangpos5 and len(FrecRange) 0.: +# self.dataOut.velocityX=numpy.append(self.dataOut.velocityX, Vzon) #Vmag +# #print 'Vmag',Vmag +# else: +# self.dataOut.velocityX=numpy.append(self.dataOut.velocityX, NaN) +# +# if abs(Vx)<100 and abs(Vx) > 0.: +# self.dataOut.velocityY=numpy.append(self.dataOut.velocityY, Vmer) #Vang +# #print 'Vang',Vang +# else: +# self.dataOut.velocityY=numpy.append(self.dataOut.velocityY, NaN) +# +# if abs(GaussCenter)<2: +# self.dataOut.velocityV=numpy.append(self.dataOut.velocityV, xFrec[Vpos]) +# +# else: +# self.dataOut.velocityV=numpy.append(self.dataOut.velocityV, NaN) +# +# +# # print '********************************************' +# # print 'HalfWidth ', HalfWidth +# # print 'Maximun ', Maximun +# # print 'eMinus1 ', eMinus1 +# # print 'Rangpos ', Rangpos +# # print 'GaussCenter ',GaussCenter +# # print 'E01 ',E01 +# # print 'N01 ',N01 +# # print 'E02 ',E02 +# # print 'N02 ',N02 +# # print 'E12 ',E12 +# # print 'N12 ',N12 +# #print 'self.dataOut.velocityX ', self.dataOut.velocityX +# # print 'Fij ', Fij +# # print 'cC ', cC +# # print 'cF ', cF +# # print 'cG ', cG +# # print 'cA ', cA +# # print 'cB ', cB +# # print 'cH ', cH +# # print 'Vx ', Vx +# # print 'Vy ', Vy +# # print 'Vmag ', Vmag +# # print 'Vang ', Vang*180/numpy.pi +# # print 'PhaseSlope ',PhaseSlope[0] +# # print 'PhaseSlope ',PhaseSlope[1] +# # print 'PhaseSlope ',PhaseSlope[2] +# # print '********************************************' +# #print 'data_output',shape(self.dataOut.velocityX), shape(self.dataOut.velocityY) +# +# #print 'self.dataOut.velocityX', len(self.dataOut.velocityX) +# #print 'self.dataOut.velocityY', len(self.dataOut.velocityY) +# #print 'self.dataOut.velocityV', self.dataOut.velocityV +# +# self.data_output[0]=numpy.array(self.dataOut.velocityX) +# self.data_output[1]=numpy.array(self.dataOut.velocityY) +# self.data_output[2]=numpy.array(self.dataOut.velocityV) +# +# prin= self.data_output[0][~numpy.isnan(self.data_output[0])] +# print ' ' +# print 'VmagAverage',numpy.mean(prin) +# print ' ' +# # plt.figure(5) +# # plt.subplot(211) +# # plt.plot(self.dataOut.velocityX,'yo:') +# # plt.subplot(212) +# # plt.plot(self.dataOut.velocityY,'yo:') +# +# # plt.figure(1) +# # # plt.subplot(121) +# # # plt.plot(xFrec,ySamples[0],'k',label='Ch0') +# # # plt.plot(xFrec,ySamples[1],'g',label='Ch1') +# # # plt.plot(xFrec,ySamples[2],'r',label='Ch2') +# # # plt.plot(xFrec,FitGauss,'yo:',label='fit') +# # # plt.legend() +# # plt.title('DATOS A ALTURA DE 2850 METROS') +# # +# # plt.xlabel('Frecuencia (KHz)') +# # plt.ylabel('Magnitud') +# # # plt.subplot(122) +# # # plt.title('Fit for Time Constant') +# # #plt.plot(xFrec,zline) +# # #plt.plot(xFrec,SmoothSPC,'g') +# # plt.plot(xFrec,FactNorm) +# # plt.axis([-4, 4, 0, 0.15]) +# # # plt.xlabel('SelfSpectra KHz') +# # +# # plt.figure(10) +# # # plt.subplot(121) +# # plt.plot(xFrec,ySamples[0],'b',label='Ch0') +# # plt.plot(xFrec,ySamples[1],'y',label='Ch1') +# # plt.plot(xFrec,ySamples[2],'r',label='Ch2') +# # # plt.plot(xFrec,FitGauss,'yo:',label='fit') +# # plt.legend() +# # plt.title('SELFSPECTRA EN CANALES') +# # +# # plt.xlabel('Frecuencia (KHz)') +# # plt.ylabel('Magnitud') +# # # plt.subplot(122) +# # # plt.title('Fit for Time Constant') +# # #plt.plot(xFrec,zline) +# # #plt.plot(xFrec,SmoothSPC,'g') +# # # plt.plot(xFrec,FactNorm) +# # # plt.axis([-4, 4, 0, 0.15]) +# # # plt.xlabel('SelfSpectra KHz') +# # +# # plt.figure(9) +# # +# # +# # plt.title('DATOS SUAVIZADOS') +# # plt.xlabel('Frecuencia (KHz)') +# # plt.ylabel('Magnitud') +# # plt.plot(xFrec,SmoothSPC,'g') +# # +# # #plt.plot(xFrec,FactNorm) +# # plt.axis([-4, 4, 0, 0.15]) +# # # plt.xlabel('SelfSpectra KHz') +# # # +# # plt.figure(2) +# # # #plt.subplot(121) +# # plt.plot(xFrec,yMean,'r',label='Mean SelfSpectra') +# # plt.plot(xFrec,FitGauss,'yo:',label='Ajuste Gaussiano') +# # # plt.plot(xFrec[Rangpos],FitGauss[Find(FitGauss,min(FitGauss, key=lambda value:abs(value-Maximun*0.1)))],'bo') +# # # #plt.plot(xFrec,phase) +# # # plt.xlabel('Suavizado, promediado KHz') +# # plt.title('SELFSPECTRA PROMEDIADO') +# # # #plt.subplot(122) +# # # #plt.plot(xSamples,zline) +# # plt.xlabel('Frecuencia (KHz)') +# # plt.ylabel('Magnitud') +# # plt.legend() +# # # +# # # plt.figure(3) +# # # plt.subplot(311) +# # # #plt.plot(xFrec,phase[0]) +# # # plt.plot(xFrec,phase[0],'g') +# # # plt.subplot(312) +# # # plt.plot(xFrec,phase[1],'g') +# # # plt.subplot(313) +# # # plt.plot(xFrec,phase[2],'g') +# # # #plt.plot(xFrec,phase[2]) +# # # +# # # plt.figure(4) +# # # +# # # plt.plot(xSamples,coherence[0],'b') +# # # plt.plot(xSamples,coherence[1],'r') +# # # plt.plot(xSamples,coherence[2],'g') +# # plt.show() +# # # +# # # plt.clf() +# # # plt.cla() +# # # plt.close() +# +# print ' ' + + + + self.BlockCounter+=2 + + else: + self.fileSelector+=1 + self.BlockCounter=0 + print "Next File" + + + + + diff --git a/schainpy/model/io/jroIO_madrigal.py b/schainpy/model/io/jroIO_madrigal.py new file mode 100644 index 0000000..7a62ffe --- /dev/null +++ b/schainpy/model/io/jroIO_madrigal.py @@ -0,0 +1,632 @@ +''' +Created on Aug 1, 2017 + +@author: Juan C. Espinoza +''' + +import os +import sys +import time +import json +import glob +import datetime + +import numpy +import h5py + +from schainpy.model.io.jroIO_base import JRODataReader +from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation +from schainpy.model.data.jrodata import Parameters +from schainpy.utils import log + +try: + import madrigal.cedar +except: + log.warning( + 'You should install "madrigal library" module if you want to read/write Madrigal data' + ) + +DEF_CATALOG = { + 'principleInvestigator': 'Marco Milla', + 'expPurpose': None, + 'cycleTime': None, + 'correlativeExp': None, + 'sciRemarks': None, + 'instRemarks': None + } +DEF_HEADER = { + 'kindatDesc': None, + 'analyst': 'Jicamarca User', + 'comments': None, + 'history': None + } +MNEMONICS = { + 10: 'jro', + 11: 'jbr', + 840: 'jul', + 13: 'jas', + 1000: 'pbr', + 1001: 'hbr', + 1002: 'obr', +} + +UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone) + +def load_json(obj): + ''' + Parse json as string instead of unicode + ''' + + if isinstance(obj, str): + iterable = json.loads(obj) + else: + iterable = obj + + if isinstance(iterable, dict): + return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, unicode) else v + for k, v in iterable.items()} + elif isinstance(iterable, (list, tuple)): + return [str(v) if isinstance(v, unicode) else v for v in iterable] + + return iterable + + +class MADReader(JRODataReader, ProcessingUnit): + + def __init__(self, **kwargs): + + ProcessingUnit.__init__(self, **kwargs) + + self.dataOut = Parameters() + self.counter_records = 0 + self.nrecords = None + self.flagNoMoreFiles = 0 + self.isConfig = False + self.filename = None + self.intervals = set() + + def setup(self, + path=None, + startDate=None, + endDate=None, + format=None, + startTime=datetime.time(0, 0, 0), + endTime=datetime.time(23, 59, 59), + **kwargs): + + self.path = path + self.startDate = startDate + self.endDate = endDate + self.startTime = startTime + self.endTime = endTime + self.datatime = datetime.datetime(1900,1,1) + self.oneDDict = load_json(kwargs.get('oneDDict', + "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}")) + self.twoDDict = load_json(kwargs.get('twoDDict', + "{\"GDALT\": \"heightList\"}")) + self.ind2DList = load_json(kwargs.get('ind2DList', + "[\"GDALT\"]")) + if self.path is None: + raise ValueError, 'The path is not valid' + + if format is None: + raise ValueError, 'The format is not valid choose simple or hdf5' + elif format.lower() in ('simple', 'txt'): + self.ext = '.txt' + elif format.lower() in ('cedar',): + self.ext = '.001' + else: + self.ext = '.hdf5' + + self.search_files(self.path) + self.fileId = 0 + + if not self.fileList: + raise Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path) + + self.setNextFile() + + def search_files(self, path): + ''' + Searching for madrigal files in path + Creating a list of files to procces included in [startDate,endDate] + + Input: + path - Path to find files + ''' + + log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader') + foldercounter = 0 + fileList0 = glob.glob1(path, '*{}'.format(self.ext)) + fileList0.sort() + + self.fileList = [] + self.dateFileList = [] + + startDate = self.startDate - datetime.timedelta(1) + endDate = self.endDate + datetime.timedelta(1) + + for thisFile in fileList0: + year = thisFile[3:7] + if not year.isdigit(): + continue + + month = thisFile[7:9] + if not month.isdigit(): + continue + + day = thisFile[9:11] + if not day.isdigit(): + continue + + year, month, day = int(year), int(month), int(day) + dateFile = datetime.date(year, month, day) + + if (startDate > dateFile) or (endDate < dateFile): + continue + + self.fileList.append(thisFile) + self.dateFileList.append(dateFile) + + return + + def parseHeader(self): + ''' + ''' + + self.output = {} + self.version = '2' + s_parameters = None + if self.ext == '.txt': + self.parameters = [s.strip().lower() for s in self.fp.readline().strip().split(' ') if s] + elif self.ext == '.hdf5': + metadata = self.fp['Metadata'] + data = self.fp['Data']['Array Layout'] + if 'Independent Spatial Parameters' in metadata: + s_parameters = [s[0].lower() for s in metadata['Independent Spatial Parameters']] + self.version = '3' + one = [s[0].lower() for s in data['1D Parameters']['Data Parameters']] + one_d = [1 for s in one] + two = [s[0].lower() for s in data['2D Parameters']['Data Parameters']] + two_d = [2 for s in two] + self.parameters = one + two + self.parameters_d = one_d + two_d + + log.success('Parameters found: {}'.format(','.join(self.parameters)), + 'MADReader') + if s_parameters: + log.success('Spatial parameters: {}'.format(','.join(s_parameters)), + 'MADReader') + + for param in self.oneDDict.keys(): + if param.lower() not in self.parameters: + log.warning( + 'Parameter {} not found will be ignored'.format( + param), + 'MADReader') + self.oneDDict.pop(param, None) + + for param, value in self.twoDDict.items(): + if param.lower() not in self.parameters: + log.warning( + 'Parameter {} not found, it will be ignored'.format( + param), + 'MADReader') + self.twoDDict.pop(param, None) + continue + if isinstance(value, list): + if value[0] not in self.output: + self.output[value[0]] = [] + self.output[value[0]].append(None) + + def parseData(self): + ''' + ''' + + if self.ext == '.txt': + self.data = numpy.genfromtxt(self.fp, missing_values=('missing')) + self.nrecords = self.data.shape[0] + self.ranges = numpy.unique(self.data[:,self.parameters.index(self.ind2DList[0].lower())]) + elif self.ext == '.hdf5': + self.data = self.fp['Data']['Array Layout'] + self.nrecords = len(self.data['timestamps'].value) + self.ranges = self.data['range'].value + + def setNextFile(self): + ''' + ''' + + file_id = self.fileId + + if file_id == len(self.fileList): + log.success('No more files', 'MADReader') + self.flagNoMoreFiles = 1 + return 0 + + log.success( + 'Opening: {}'.format(self.fileList[file_id]), + 'MADReader' + ) + + filename = os.path.join(self.path, self.fileList[file_id]) + + if self.filename is not None: + self.fp.close() + + self.filename = filename + self.filedate = self.dateFileList[file_id] + + if self.ext=='.hdf5': + self.fp = h5py.File(self.filename, 'r') + else: + self.fp = open(self.filename, 'rb') + + self.parseHeader() + self.parseData() + self.sizeOfFile = os.path.getsize(self.filename) + self.counter_records = 0 + self.flagIsNewFile = 0 + self.fileId += 1 + + return 1 + + def readNextBlock(self): + + while True: + self.flagDiscontinuousBlock = 0 + if self.flagIsNewFile: + if not self.setNextFile(): + return 0 + + self.readBlock() + + if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \ + (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)): + log.warning( + 'Reading Record No. {}/{} -> {} [Skipping]'.format( + self.counter_records, + self.nrecords, + self.datatime.ctime()), + 'MADReader') + continue + break + + log.log( + 'Reading Record No. {}/{} -> {}'.format( + self.counter_records, + self.nrecords, + self.datatime.ctime()), + 'MADReader') + + return 1 + + def readBlock(self): + ''' + ''' + dum = [] + if self.ext == '.txt': + dt = self.data[self.counter_records][:6].astype(int) + self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]) + while True: + dt = self.data[self.counter_records][:6].astype(int) + datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]) + if datatime == self.datatime: + dum.append(self.data[self.counter_records]) + self.counter_records += 1 + if self.counter_records == self.nrecords: + self.flagIsNewFile = True + break + continue + self.intervals.add((datatime-self.datatime).seconds) + if datatime.date() > self.datatime.date(): + self.flagDiscontinuousBlock = 1 + break + elif self.ext == '.hdf5': + datatime = datetime.datetime.utcfromtimestamp( + self.data['timestamps'][self.counter_records]) + nHeights = len(self.ranges) + for n, param in enumerate(self.parameters): + if self.parameters_d[n] == 1: + dum.append(numpy.ones(nHeights)*self.data['1D Parameters'][param][self.counter_records]) + else: + if self.version == '2': + dum.append(self.data['2D Parameters'][param][self.counter_records]) + else: + tmp = self.data['2D Parameters'][param].value.T + dum.append(tmp[self.counter_records]) + self.intervals.add((datatime-self.datatime).seconds) + if datatime.date()>self.datatime.date(): + self.flagDiscontinuousBlock = 1 + self.datatime = datatime + self.counter_records += 1 + if self.counter_records == self.nrecords: + self.flagIsNewFile = True + + self.buffer = numpy.array(dum) + return + + def set_output(self): + ''' + Storing data from buffer to dataOut object + ''' + + parameters = [None for __ in self.parameters] + + for param, attr in self.oneDDict.items(): + x = self.parameters.index(param.lower()) + setattr(self.dataOut, attr, self.buffer[0][x]) + + for param, value in self.twoDDict.items(): + x = self.parameters.index(param.lower()) + if self.ext == '.txt': + y = self.parameters.index(self.ind2DList[0].lower()) + ranges = self.buffer[:,y] + if self.ranges.size == ranges.size: + continue + index = numpy.where(numpy.in1d(self.ranges, ranges))[0] + dummy = numpy.zeros(self.ranges.shape) + numpy.nan + dummy[index] = self.buffer[:,x] + else: + dummy = self.buffer[x] + + if isinstance(value, str): + if value not in self.ind2DList: + setattr(self.dataOut, value, dummy.reshape(1,-1)) + elif isinstance(value, list): + self.output[value[0]][value[1]] = dummy + parameters[value[1]] = param + + for key, value in self.output.items(): + setattr(self.dataOut, key, numpy.array(value)) + + self.dataOut.parameters = [s for s in parameters if s] + self.dataOut.heightList = self.ranges + self.dataOut.utctime = (self.datatime - UT1970).total_seconds() + self.dataOut.utctimeInit = self.dataOut.utctime + self.dataOut.paramInterval = min(self.intervals) + self.dataOut.useLocalTime = False + self.dataOut.flagNoData = False + self.dataOut.nrecords = self.nrecords + self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock + + def getData(self): + ''' + Storing data from databuffer to dataOut object + ''' + if self.flagNoMoreFiles: + self.dataOut.flagNoData = True + log.error('No file left to process', 'MADReader') + return 0 + + if not self.readNextBlock(): + self.dataOut.flagNoData = True + return 0 + + self.set_output() + + return 1 + + +class MADWriter(Operation): + + missing = -32767 + + def __init__(self, **kwargs): + + Operation.__init__(self, **kwargs) + self.dataOut = Parameters() + self.path = None + self.fp = None + + def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}', + metadata='{}', format='cedar', **kwargs): + ''' + Inputs: + path - path where files will be created + oneDDict - json of one-dimensional parameters in record where keys + are Madrigal codes (integers or mnemonics) and values the corresponding + dataOut attribute e.g: { + 'gdlatr': 'lat', + 'gdlonr': 'lon', + 'gdlat2':'lat', + 'glon2':'lon'} + ind2DList - list of independent spatial two-dimensional parameters e.g: + ['heighList'] + twoDDict - json of two-dimensional parameters in record where keys + are Madrigal codes (integers or mnemonics) and values the corresponding + dataOut attribute if multidimensional array specify as tupple + ('attr', pos) e.g: { + 'gdalt': 'heightList', + 'vn1p2': ('data_output', 0), + 'vn2p2': ('data_output', 1), + 'vn3': ('data_output', 2), + 'snl': ('data_SNR', 'db') + } + metadata - json of madrigal metadata (kinst, kindat, catalog and header) + ''' + if not self.isConfig: + self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs) + self.isConfig = True + + self.dataOut = dataOut + self.putData() + return + + def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs): + ''' + Configure Operation + ''' + + self.path = path + self.blocks = kwargs.get('blocks', None) + self.counter = 0 + self.oneDDict = load_json(oneDDict) + self.twoDDict = load_json(twoDDict) + self.ind2DList = load_json(ind2DList) + meta = load_json(metadata) + self.kinst = meta.get('kinst') + self.kindat = meta.get('kindat') + self.catalog = meta.get('catalog', DEF_CATALOG) + self.header = meta.get('header', DEF_HEADER) + if format == 'cedar': + self.ext = '.dat' + self.extra_args = {} + elif format == 'hdf5': + self.ext = '.hdf5' + self.extra_args = {'ind2DList': self.ind2DList} + + self.keys = [k.lower() for k in self.twoDDict] + if 'range' in self.keys: + self.keys.remove('range') + if 'gdalt' in self.keys: + self.keys.remove('gdalt') + + def setFile(self): + ''' + Create new cedar file object + ''' + + self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal + date = datetime.datetime.fromtimestamp(self.dataOut.utctime) + + filename = '{}{}{}'.format(self.mnemonic, + date.strftime('%Y%m%d_%H%M%S'), + self.ext) + + self.fullname = os.path.join(self.path, filename) + + if os.path.isfile(self.fullname) : + log.warning( + 'Destination path {} already exists. Previous file deleted.'.format( + self.fullname), + 'MADWriter') + os.remove(self.fullname) + + try: + log.success( + 'Creating file: {}'.format(self.fullname), + 'MADWriter') + self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True) + except ValueError, e: + log.error( + 'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"', + 'MADWriter') + return + + return 1 + + def writeBlock(self): + ''' + Add data records to cedar file taking data from oneDDict and twoDDict + attributes. + Allowed parameters in: parcodes.tab + ''' + + startTime = datetime.datetime.fromtimestamp(self.dataOut.utctime) + endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval) + heights = self.dataOut.heightList + + if self.ext == '.dat': + invalid = numpy.isnan(self.dataOut.data_output) + self.dataOut.data_output[invalid] = self.missing + out = {} + for key, value in self.twoDDict.items(): + key = key.lower() + if isinstance(value, str): + if 'db' in value.lower(): + tmp = getattr(self.dataOut, value.replace('_db', '')) + SNRavg = numpy.average(tmp, axis=0) + tmp = 10*numpy.log10(SNRavg) + else: + tmp = getattr(self.dataOut, value) + out[key] = tmp.flatten() + elif isinstance(value, (tuple, list)): + attr, x = value + data = getattr(self.dataOut, attr) + out[key] = data[int(x)] + + a = numpy.array([out[k] for k in self.keys]) + nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))]) + index = numpy.where(nrows == False)[0] + + rec = madrigal.cedar.MadrigalDataRecord( + self.kinst, + self.kindat, + startTime.year, + startTime.month, + startTime.day, + startTime.hour, + startTime.minute, + startTime.second, + startTime.microsecond/10000, + endTime.year, + endTime.month, + endTime.day, + endTime.hour, + endTime.minute, + endTime.second, + endTime.microsecond/10000, + self.oneDDict.keys(), + self.twoDDict.keys(), + len(index), + **self.extra_args + ) + + # Setting 1d values + for key in self.oneDDict: + rec.set1D(key, getattr(self.dataOut, self.oneDDict[key])) + + # Setting 2d values + nrec = 0 + for n in index: + for key in out: + rec.set2D(key, nrec, out[key][n]) + nrec += 1 + + self.fp.append(rec) + if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0: + self.fp.dump() + if self.counter % 10 == 0 and self.counter > 0: + log.log( + 'Writing {} records'.format( + self.counter), + 'MADWriter') + + def setHeader(self): + ''' + Create an add catalog and header to cedar file + ''' + + log.success('Closing file {}'.format(self.fullname), 'MADWriter') + + if self.ext == '.dat': + self.fp.write() + else: + self.fp.dump() + self.fp.close() + + header = madrigal.cedar.CatalogHeaderCreator(self.fullname) + header.createCatalog(**self.catalog) + header.createHeader(**self.header) + header.write() + + def putData(self): + + if self.dataOut.flagNoData: + return 0 + + if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks: + if self.counter > 0: + self.setHeader() + self.counter = 0 + + if self.counter == 0: + self.setFile() + + self.writeBlock() + self.counter += 1 + + def close(self): + + if self.counter > 0: + self.setHeader() diff --git a/schainpy/model/io/jroIO_mira35c.py b/schainpy/model/io/jroIO_mira35c.py new file mode 100644 index 0000000..632bc93 --- /dev/null +++ b/schainpy/model/io/jroIO_mira35c.py @@ -0,0 +1,803 @@ +import os, sys +import glob +import fnmatch +import datetime +import time +import re +import h5py +import numpy +import matplotlib.pyplot as plt + +import pylab as plb +from scipy.optimize import curve_fit +from scipy import asarray as ar,exp +from scipy import stats + +from numpy.ma.core import getdata + +SPEED_OF_LIGHT = 299792458 +SPEED_OF_LIGHT = 3e8 + +try: + from gevent import sleep +except: + from time import sleep + +from schainpy.model.data.jrodata import Spectra +#from schainpy.model.data.BLTRheaderIO import FileHeader, RecordHeader +from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation +#from schainpy.model.io.jroIO_bltr import BLTRReader +from numpy import imag, shape, NaN, empty + + + +class Header(object): + + def __init__(self): + raise NotImplementedError + + + def read(self): + + raise NotImplementedError + + def write(self): + + raise NotImplementedError + + def printInfo(self): + + message = "#"*50 + "\n" + message += self.__class__.__name__.upper() + "\n" + message += "#"*50 + "\n" + + keyList = self.__dict__.keys() + keyList.sort() + + for key in keyList: + message += "%s = %s" %(key, self.__dict__[key]) + "\n" + + if "size" not in keyList: + attr = getattr(self, "size") + + if attr: + message += "%s = %s" %("size", attr) + "\n" + + #print message + + +FILE_HEADER = numpy.dtype([ #HEADER 1024bytes + ('Hname','a32'), #Original file name + ('Htime',numpy.str_,32), #Date and time when the file was created + ('Hoper',numpy.str_,64), #Name of operator who created the file + ('Hplace',numpy.str_,128), #Place where the measurements was carried out + ('Hdescr',numpy.str_,256), #Description of measurements + ('Hdummy',numpy.str_,512), #Reserved space + #Main chunk 8bytes + ('Msign',numpy.str_,4), #Main chunk signature FZKF or NUIG + ('MsizeData','=5 + ('SPARrawGate2',' 1180: + self.fp.seek(self.PointerReader , os.SEEK_SET) + self.FirstPoint = self.PointerReader + + else : + self.FirstPoint = 1180 + + + + self.srviHeader = SRVIHeader() + + self.srviHeader.SRVIread(self.fp) #Se obtiene la cabecera del SRVI + + self.blocksize = self.srviHeader.SizeOfDataBlock1 # Se obtiene el tamao del bloque + + if self.blocksize == 148: + print 'blocksize == 148 bug' + jump = numpy.fromfile(self.fp,[('jump',numpy.str_,140)] ,1) + + self.srviHeader.SRVIread(self.fp) #Se obtiene la cabecera del SRVI + + if not self.srviHeader.SizeOfSRVI1: + self.fileSelector+=1 + self.nextfileflag==True + self.FileHeaderFlag == True + + self.recordheader = RecordHeader() + self.recordheader.RHread(self.fp) + self.RadarConst = self.recordheader.RadarConst + dwell = self.recordheader.time_t + npw1 = self.recordheader.npw1 + npw2 = self.recordheader.npw2 + + + self.dataOut.channelList = range(1) + self.dataOut.nIncohInt = self.Num_inCoh + self.dataOut.nProfiles = self.Num_Bins + self.dataOut.nCohInt = 1 + self.dataOut.windowOfFilter = 1 + self.dataOut.utctime = dwell + self.dataOut.timeZone=0 + + self.dataOut.outputInterval = self.dataOut.getTimeInterval() + self.dataOut.heightList = self.SPARrawGate1*self.__deltaHeigth + numpy.array(range(self.Num_Hei))*self.__deltaHeigth + + + + self.HSDVsign = numpy.fromfile( self.fp, [('HSDV',numpy.str_,4)],1) + self.SizeHSDV = numpy.fromfile( self.fp, [('SizeHSDV',' 0. , self.data_spc, 0) + + self.dataOut.COFA = numpy.array([self.COFA_Co , self.COFA_Cx]) + + print ' ' + print 'SPC',numpy.shape(self.dataOut.data_spc) + #print 'SPC',self.dataOut.data_spc + + noinor1 = 713031680 + noinor2 = 30 + + npw1 = 1#0**(npw1/10) * noinor1 * noinor2 + npw2 = 1#0**(npw2/10) * noinor1 * noinor2 + self.dataOut.NPW = numpy.array([npw1, npw2]) + + print ' ' + + self.data_spc = numpy.transpose(self.data_spc, (2,1,0)) + self.data_spc = numpy.fft.fftshift(self.data_spc, axes = 1) + + self.data_spc = numpy.fliplr(self.data_spc) + + self.data_spc = numpy.where(self.data_spc > 0. , self.data_spc, 0) + self.dataOut_spc= numpy.ones([1, self.Num_Bins , self.Num_Hei]) + self.dataOut_spc[0,:,:] = self.data_spc[0,:,:] + #print 'SHAPE', self.dataOut_spc.shape + #For nyquist correction: + #fix = 20 # ~3m/s + #shift = self.Num_Bins/2 + fix + #self.data_spc = numpy.array([ self.data_spc[: , self.Num_Bins-shift+1: , :] , self.data_spc[: , 0:self.Num_Bins-shift , :]]) + + + + '''Block Reading, the Block Data is received and Reshape is used to give it + shape. + ''' + + self.PointerReader = self.fp.tell() + + + + + + + \ No newline at end of file diff --git a/schainpy/model/io/jroIO_param.py b/schainpy/model/io/jroIO_param.py index 50e7367..fa4495c 100644 --- a/schainpy/model/io/jroIO_param.py +++ b/schainpy/model/io/jroIO_param.py @@ -178,8 +178,8 @@ class ParamReader(ProcessingUnit): print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime) print - for i in range(len(filenameList)): - print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) +# for i in range(len(filenameList)): +# print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) self.filenameList = filenameList self.datetimeList = datetimeList diff --git a/schainpy/model/proc/__init__.py b/schainpy/model/proc/__init__.py index a17c2ba..af598f5 100644 --- a/schainpy/model/proc/__init__.py +++ b/schainpy/model/proc/__init__.py @@ -11,4 +11,5 @@ from jroproc_amisr import * from jroproc_correlation import * from jroproc_parameters import * from jroproc_spectra_lags import * -from jroproc_spectra_acf import * \ No newline at end of file +from jroproc_spectra_acf import * +from bltrproc_parameters import * diff --git a/schainpy/model/proc/bltrproc_parameters.py b/schainpy/model/proc/bltrproc_parameters.py new file mode 100644 index 0000000..cada61c --- /dev/null +++ b/schainpy/model/proc/bltrproc_parameters.py @@ -0,0 +1,403 @@ +''' +Created on Oct 24, 2016 + +@author: roj- LouVD +''' + +import numpy +import copy +import datetime +import time +from time import gmtime + +from numpy import transpose + +from jroproc_base import ProcessingUnit, Operation +from schainpy.model.data.jrodata import Parameters + + +class BLTRParametersProc(ProcessingUnit): + ''' + Processing unit for BLTR parameters data (winds) + + Inputs: + self.dataOut.nmodes - Number of operation modes + self.dataOut.nchannels - Number of channels + self.dataOut.nranges - Number of ranges + + self.dataOut.data_SNR - SNR array + self.dataOut.data_output - Zonal, Vertical and Meridional velocity array + self.dataOut.height - Height array (km) + self.dataOut.time - Time array (seconds) + + self.dataOut.fileIndex -Index of the file currently read + self.dataOut.lat - Latitude coordinate of BLTR location + + self.dataOut.doy - Experiment doy (number of the day in the current year) + self.dataOut.month - Experiment month + self.dataOut.day - Experiment day + self.dataOut.year - Experiment year + ''' + + def __init__(self, **kwargs): + ''' + Inputs: None + ''' + ProcessingUnit.__init__(self, **kwargs) + self.dataOut = Parameters() + self.isConfig = False + + def setup(self, mode): + ''' + ''' + self.dataOut.mode = mode + + def run(self, mode, snr_threshold=None): + ''' + Inputs: + mode = High resolution (0) or Low resolution (1) data + snr_threshold = snr filter value + ''' + + if not self.isConfig: + self.setup(mode) + self.isConfig = True + + if self.dataIn.type == 'Parameters': + self.dataOut.copy(self.dataIn) + + self.dataOut.data_output = self.dataOut.data_output[mode] + self.dataOut.heightList = self.dataOut.height[0] + self.dataOut.data_SNR = self.dataOut.data_SNR[mode] + + if snr_threshold is not None: + SNRavg = numpy.average(self.dataOut.data_SNR, axis=0) + SNRavgdB = 10*numpy.log10(SNRavg) + for i in range(3): + self.dataOut.data_output[i][SNRavgdB <= snr_threshold] = numpy.nan + +# TODO +class OutliersFilter(Operation): + + def __init__(self, **kwargs): + ''' + ''' + Operation.__init__(self, **kwargs) + + def run(self, svalue2, method, factor, filter, npoints=9): + ''' + Inputs: + svalue - string to select array velocity + svalue2 - string to choose axis filtering + method - 0 for SMOOTH or 1 for MEDIAN + factor - number used to set threshold + filter - 1 for data filtering using the standard deviation criteria else 0 + npoints - number of points for mask filter + ''' + + print ' Outliers Filter {} {} / threshold = {}'.format(svalue, svalue, factor) + + + yaxis = self.dataOut.heightList + xaxis = numpy.array([[self.dataOut.utctime]]) + + # Zonal + value_temp = self.dataOut.data_output[0] + + # Zonal + value_temp = self.dataOut.data_output[1] + + # Vertical + value_temp = numpy.transpose(self.dataOut.data_output[2]) + + htemp = yaxis + std = value_temp + for h in range(len(htemp)): + nvalues_valid = len(numpy.where(numpy.isfinite(value_temp[h]))[0]) + minvalid = npoints + + #only if valid values greater than the minimum required (10%) + if nvalues_valid > minvalid: + + if method == 0: + #SMOOTH + w = value_temp[h] - self.Smooth(input=value_temp[h], width=npoints, edge_truncate=1) + + + if method == 1: + #MEDIAN + w = value_temp[h] - self.Median(input=value_temp[h], width = npoints) + + dw = numpy.std(w[numpy.where(numpy.isfinite(w))],ddof = 1) + + threshold = dw*factor + value_temp[numpy.where(w > threshold),h] = numpy.nan + value_temp[numpy.where(w < -1*threshold),h] = numpy.nan + + + #At the end + if svalue2 == 'inHeight': + value_temp = numpy.transpose(value_temp) + output_array[:,m] = value_temp + + if svalue == 'zonal': + self.dataOut.data_output[0] = output_array + + elif svalue == 'meridional': + self.dataOut.data_output[1] = output_array + + elif svalue == 'vertical': + self.dataOut.data_output[2] = output_array + + return self.dataOut.data_output + + + def Median(self,input,width): + ''' + Inputs: + input - Velocity array + width - Number of points for mask filter + + ''' + + if numpy.mod(width,2) == 1: + pc = int((width - 1) / 2) + cont = 0 + output = [] + + for i in range(len(input)): + if i >= pc and i < len(input) - pc: + new2 = input[i-pc:i+pc+1] + temp = numpy.where(numpy.isfinite(new2)) + new = new2[temp] + value = numpy.median(new) + output.append(value) + + output = numpy.array(output) + output = numpy.hstack((input[0:pc],output)) + output = numpy.hstack((output,input[-pc:len(input)])) + + return output + + def Smooth(self,input,width,edge_truncate = None): + ''' + Inputs: + input - Velocity array + width - Number of points for mask filter + edge_truncate - 1 for truncate the convolution product else + + ''' + + if numpy.mod(width,2) == 0: + real_width = width + 1 + nzeros = width / 2 + else: + real_width = width + nzeros = (width - 1) / 2 + + half_width = int(real_width)/2 + length = len(input) + + gate = numpy.ones(real_width,dtype='float') + norm_of_gate = numpy.sum(gate) + + nan_process = 0 + nan_id = numpy.where(numpy.isnan(input)) + if len(nan_id[0]) > 0: + nan_process = 1 + pb = numpy.zeros(len(input)) + pb[nan_id] = 1. + input[nan_id] = 0. + + if edge_truncate == True: + output = numpy.convolve(input/norm_of_gate,gate,mode='same') + elif edge_truncate == False or edge_truncate == None: + output = numpy.convolve(input/norm_of_gate,gate,mode='valid') + output = numpy.hstack((input[0:half_width],output)) + output = numpy.hstack((output,input[len(input)-half_width:len(input)])) + + if nan_process: + pb = numpy.convolve(pb/norm_of_gate,gate,mode='valid') + pb = numpy.hstack((numpy.zeros(half_width),pb)) + pb = numpy.hstack((pb,numpy.zeros(half_width))) + output[numpy.where(pb > 0.9999)] = numpy.nan + input[nan_id] = numpy.nan + return output + + def Average(self,aver=0,nhaver=1): + ''' + Inputs: + aver - Indicates the time period over which is averaged or consensus data + nhaver - Indicates the decimation factor in heights + + ''' + nhpoints = 48 + + lat_piura = -5.17 + lat_huancayo = -12.04 + lat_porcuya = -5.8 + + if '%2.2f'%self.dataOut.lat == '%2.2f'%lat_piura: + hcm = 3. + if self.dataOut.year == 2003 : + if self.dataOut.doy >= 25 and self.dataOut.doy < 64: + nhpoints = 12 + + elif '%2.2f'%self.dataOut.lat == '%2.2f'%lat_huancayo: + hcm = 3. + if self.dataOut.year == 2003 : + if self.dataOut.doy >= 25 and self.dataOut.doy < 64: + nhpoints = 12 + + + elif '%2.2f'%self.dataOut.lat == '%2.2f'%lat_porcuya: + hcm = 5.#2 + + pdata = 0.2 + taver = [1,2,3,4,6,8,12,24] + t0 = 0 + tf = 24 + ntime =(tf-t0)/taver[aver] + ti = numpy.arange(ntime) + tf = numpy.arange(ntime) + taver[aver] + + + old_height = self.dataOut.heightList + + if nhaver > 1: + num_hei = len(self.dataOut.heightList)/nhaver/self.dataOut.nmodes + deltha = 0.05*nhaver + minhvalid = pdata*nhaver + for im in range(self.dataOut.nmodes): + new_height = numpy.arange(num_hei)*deltha + self.dataOut.height[im,0] + deltha/2. + + + data_fHeigths_List = [] + data_fZonal_List = [] + data_fMeridional_List = [] + data_fVertical_List = [] + startDTList = [] + + + for i in range(ntime): + height = old_height + + start = datetime.datetime(self.dataOut.year,self.dataOut.month,self.dataOut.day) + datetime.timedelta(hours = int(ti[i])) - datetime.timedelta(hours = 5) + stop = datetime.datetime(self.dataOut.year,self.dataOut.month,self.dataOut.day) + datetime.timedelta(hours = int(tf[i])) - datetime.timedelta(hours = 5) + + + limit_sec1 = time.mktime(start.timetuple()) + limit_sec2 = time.mktime(stop.timetuple()) + + t1 = numpy.where(self.f_timesec >= limit_sec1) + t2 = numpy.where(self.f_timesec < limit_sec2) + time_select = [] + for val_sec in t1[0]: + if val_sec in t2[0]: + time_select.append(val_sec) + + + time_select = numpy.array(time_select,dtype = 'int') + minvalid = numpy.ceil(pdata*nhpoints) + + zon_aver = numpy.zeros([self.dataOut.nranges,self.dataOut.nmodes],dtype='f4') + numpy.nan + mer_aver = numpy.zeros([self.dataOut.nranges,self.dataOut.nmodes],dtype='f4') + numpy.nan + ver_aver = numpy.zeros([self.dataOut.nranges,self.dataOut.nmodes],dtype='f4') + numpy.nan + + if nhaver > 1: + new_zon_aver = numpy.zeros([num_hei,self.dataOut.nmodes],dtype='f4') + numpy.nan + new_mer_aver = numpy.zeros([num_hei,self.dataOut.nmodes],dtype='f4') + numpy.nan + new_ver_aver = numpy.zeros([num_hei,self.dataOut.nmodes],dtype='f4') + numpy.nan + + if len(time_select) > minvalid: + time_average = self.f_timesec[time_select] + + for im in range(self.dataOut.nmodes): + + for ih in range(self.dataOut.nranges): + if numpy.sum(numpy.isfinite(self.f_zon[time_select,ih,im])) >= minvalid: + zon_aver[ih,im] = numpy.nansum(self.f_zon[time_select,ih,im]) / numpy.sum(numpy.isfinite(self.f_zon[time_select,ih,im])) + + if numpy.sum(numpy.isfinite(self.f_mer[time_select,ih,im])) >= minvalid: + mer_aver[ih,im] = numpy.nansum(self.f_mer[time_select,ih,im]) / numpy.sum(numpy.isfinite(self.f_mer[time_select,ih,im])) + + if numpy.sum(numpy.isfinite(self.f_ver[time_select,ih,im])) >= minvalid: + ver_aver[ih,im] = numpy.nansum(self.f_ver[time_select,ih,im]) / numpy.sum(numpy.isfinite(self.f_ver[time_select,ih,im])) + + if nhaver > 1: + for ih in range(num_hei): + hvalid = numpy.arange(nhaver) + nhaver*ih + + if numpy.sum(numpy.isfinite(zon_aver[hvalid,im])) >= minvalid: + new_zon_aver[ih,im] = numpy.nansum(zon_aver[hvalid,im]) / numpy.sum(numpy.isfinite(zon_aver[hvalid,im])) + + if numpy.sum(numpy.isfinite(mer_aver[hvalid,im])) >= minvalid: + new_mer_aver[ih,im] = numpy.nansum(mer_aver[hvalid,im]) / numpy.sum(numpy.isfinite(mer_aver[hvalid,im])) + + if numpy.sum(numpy.isfinite(ver_aver[hvalid,im])) >= minvalid: + new_ver_aver[ih,im] = numpy.nansum(ver_aver[hvalid,im]) / numpy.sum(numpy.isfinite(ver_aver[hvalid,im])) + if nhaver > 1: + zon_aver = new_zon_aver + mer_aver = new_mer_aver + ver_aver = new_ver_aver + height = new_height + + + tstart = time_average[0] + tend = time_average[-1] + startTime = time.gmtime(tstart) + + year = startTime.tm_year + month = startTime.tm_mon + day = startTime.tm_mday + hour = startTime.tm_hour + minute = startTime.tm_min + second = startTime.tm_sec + + startDTList.append(datetime.datetime(year,month,day,hour,minute,second)) + + + o_height = numpy.array([]) + o_zon_aver = numpy.array([]) + o_mer_aver = numpy.array([]) + o_ver_aver = numpy.array([]) + if self.dataOut.nmodes > 1: + for im in range(self.dataOut.nmodes): + + if im == 0: + h_select = numpy.where(numpy.bitwise_and(height[0,:] >=0,height[0,:] <= hcm,numpy.isfinite(height[0,:]))) + else: + h_select = numpy.where(numpy.bitwise_and(height[1,:] > hcm,height[1,:] < 20,numpy.isfinite(height[1,:]))) + + + ht = h_select[0] + + o_height = numpy.hstack((o_height,height[im,ht])) + o_zon_aver = numpy.hstack((o_zon_aver,zon_aver[ht,im])) + o_mer_aver = numpy.hstack((o_mer_aver,mer_aver[ht,im])) + o_ver_aver = numpy.hstack((o_ver_aver,ver_aver[ht,im])) + + data_fHeigths_List.append(o_height) + data_fZonal_List.append(o_zon_aver) + data_fMeridional_List.append(o_mer_aver) + data_fVertical_List.append(o_ver_aver) + + + else: + h_select = numpy.where(numpy.bitwise_and(height[0,:] <= hcm,numpy.isfinite(height[0,:]))) + ht = h_select[0] + o_height = numpy.hstack((o_height,height[im,ht])) + o_zon_aver = numpy.hstack((o_zon_aver,zon_aver[ht,im])) + o_mer_aver = numpy.hstack((o_mer_aver,mer_aver[ht,im])) + o_ver_aver = numpy.hstack((o_ver_aver,ver_aver[ht,im])) + + data_fHeigths_List.append(o_height) + data_fZonal_List.append(o_zon_aver) + data_fMeridional_List.append(o_mer_aver) + data_fVertical_List.append(o_ver_aver) + + + return startDTList, data_fHeigths_List, data_fZonal_List, data_fMeridional_List, data_fVertical_List + + + \ No newline at end of file diff --git a/schainpy/model/utils/jroutils_publish.py b/schainpy/model/utils/jroutils_publish.py index 726065a..cea868b 100644 --- a/schainpy/model/utils/jroutils_publish.py +++ b/schainpy/model/utils/jroutils_publish.py @@ -113,6 +113,8 @@ class Data(object): self.__heights = [] self.__all_heights = set() for plot in self.plottypes: + if 'snr' in plot: + plot = 'snr' self.data[plot] = {} def shape(self, key): @@ -138,8 +140,9 @@ class Data(object): self.parameters = getattr(dataOut, 'parameters', []) self.pairs = dataOut.pairsList self.channels = dataOut.channelList - self.xrange = (dataOut.getFreqRange(1)/1000. , dataOut.getAcfRange(1) , dataOut.getVelRange(1)) self.interval = dataOut.getTimeInterval() + if 'spc' in self.plottypes or 'cspc' in self.plottypes: + self.xrange = (dataOut.getFreqRange(1)/1000. , dataOut.getAcfRange(1) , dataOut.getVelRange(1)) self.__heights.append(dataOut.heightList) self.__all_heights.update(dataOut.heightList) self.__times.append(tm) diff --git a/schainpy/project.py b/schainpy/project.py deleted file mode 100644 index bdea4c3..0000000 --- a/schainpy/project.py +++ /dev/null @@ -1,34 +0,0 @@ -from schainpy.controller import Project - -desc = "A schain project" - -controller = Project() -controller.setup(id='191', name="project", description=desc) - -readUnitConf = controller.addReadUnit(datatype='VoltageReader', - path="/home/nanosat/schain/schainpy", - startDate="1970/01/01", - endDate="2017/12/31", - startTime="00:00:00", - endTime="23:59:59", - online=0, - verbose=1, - walk=1, - ) - -procUnitConf1 = controller.addProcUnit(datatype='VoltageProc', inputId=readUnitConf.getId()) - -opObj11 = procUnitConf1.addOperation(name='ProfileSelector', optype='other') -opObj11.addParameter(name='profileRangeList', value='120,183', format='intlist') - -opObj11 = procUnitConf1.addOperation(name='RTIPlot', optype='other') -opObj11.addParameter(name='wintitle', value='Jicamarca Radio Observatory', format='str') -opObj11.addParameter(name='showprofile', value='0', format='int') -opObj11.addParameter(name='xmin', value='0', format='int') -opObj11.addParameter(name='xmax', value='24', format='int') -opObj11.addParameter(name='figpath', value="/home/nanosat/schain/schainpy/figs", format='str') -opObj11.addParameter(name='wr_period', value='5', format='int') -opObj11.addParameter(name='exp_code', value='22', format='int') - - -controller.start()