diff --git a/schainpy/controller.py b/schainpy/controller.py index cf357aa..bcce720 100644 --- a/schainpy/controller.py +++ b/schainpy/controller.py @@ -9,6 +9,8 @@ from xml.dom import minidom import datetime from model import * +import ast + def prettify(elem): """Return a pretty-printed XML string for the Element. """ @@ -89,6 +91,14 @@ class ParameterConf(): return pairList + if self.format == 'multiList': + """ + Example: + value = (0,1,2),(3,4,5) + """ + multiList = ast.literal_eval(value) + return multiList + func = eval(self.format) self.__value = func(value) diff --git a/schainpy/model/data/jrodata.py b/schainpy/model/data/jrodata.py index 37fa7d7..5a39c6f 100644 --- a/schainpy/model/data/jrodata.py +++ b/schainpy/model/data/jrodata.py @@ -933,23 +933,23 @@ class Parameters(JROData): data_pre = None #Data Pre Parametrization + data_SNR = None #Signal to Noise Ratio + heightRange = None #Heights abscissaRange = None #Abscissa, can be velocities, lags or time noise = None #Noise Potency - SNR = None #Signal to Noise Ratio - initUtcTime = None #Initial UTC time paramInterval = None #Time interval to calculate Parameters in seconds #Fitting - constants = None - - error = None + data_error = None #Error of the estimation + + constants = None library = None diff --git a/schainpy/model/graphics/jroplot_parameters.py b/schainpy/model/graphics/jroplot_parameters.py index ce0b319..c479390 100644 --- a/schainpy/model/graphics/jroplot_parameters.py +++ b/schainpy/model/graphics/jroplot_parameters.py @@ -460,9 +460,9 @@ class WindProfilerPlot(Figure): nplotsw = nplots #If there is a SNR function defined - if dataOut.SNR != None: + if dataOut.data_SNR != None: nplots += 1 - SNR = dataOut.SNR + SNR = dataOut.data_SNR SNRavg = numpy.average(SNR, axis=0) SNRdB = 10*numpy.log10(SNR) @@ -503,7 +503,7 @@ class WindProfilerPlot(Figure): if zmax_ver == None: zmax_ver = numpy.nanmax(abs(z[2,:])) if zmin_ver == None: zmin_ver = -zmax_ver - if dataOut.SNR != None: + if dataOut.data_SNR != None: if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB) if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB) @@ -539,7 +539,7 @@ class WindProfilerPlot(Figure): xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True, ticksize=9, cblabel=strCb[i], cbsize="1%", colormap="RdBu_r" ) - if dataOut.SNR != None: + if dataOut.data_SNR != None: i += 1 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S")) axes = self.axesList[i*self.__nsubplots] @@ -719,7 +719,7 @@ class ParametersPlot(Figure): if zmin == None: zmin = numpy.nanmin(zRange) if zmax == None: zmax = numpy.nanmax(zRange) - if dataOut.SNR != None: + if dataOut.data_SNR != None: if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB) if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB) @@ -1061,9 +1061,9 @@ class EWDriftsPlot(Figure): nplotsw = nplots #If there is a SNR function defined - if dataOut.SNR != None: + if dataOut.data_SNR != None: nplots += 1 - SNR = dataOut.SNR + SNR = dataOut.data_SNR if SNR_1: SNR += 1 @@ -1104,7 +1104,7 @@ class EWDriftsPlot(Figure): if zmaxVertical == None: zmaxVertical = numpy.nanmax(abs(z[1,:])) if zminVertical == None: zminVertical = -zmaxVertical - if dataOut.SNR != None: + if dataOut.data_SNR != None: if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB) if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB) @@ -1139,7 +1139,7 @@ class EWDriftsPlot(Figure): xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True, ticksize=9, cblabel=strCb, cbsize="1%", colormap="RdBu_r") - if dataOut.SNR != None: + if dataOut.data_SNR != None: i += 1 if SNR_1: title = "Signal Noise Ratio + 1 (SNR+1): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S")) diff --git a/schainpy/model/io/jroIO_HDF5.py b/schainpy/model/io/jroIO_HDF5.py new file mode 100644 index 0000000..e6a1436 --- /dev/null +++ b/schainpy/model/io/jroIO_HDF5.py @@ -0,0 +1,656 @@ +import numpy +import time +import os +import h5py +import re + +from model.data.jrodata import * +from model.proc.jroproc_base import ProcessingUnit, Operation +from model.io.jroIO_base import * + + +class HDF5Reader(ProcessingUnit): + + ext = ".hdf5" + + optchar = "D" + + timezone = None + + fileIndex = None + + blockIndex = None + + path = None + + #Hdf5 File + + fpMetadata = None + + listMetaname = None + + listMetadata = None + + fp = None + + #dataOut reconstruction + + + dataOut = None + + nChannels = None #Dimension 0 + + nPoints = None #Dimension 1, number of Points or Parameters + + nSamples = None #Dimension 2, number of samples or ranges + + + def __init__(self): + + return + + def setup(self,path=None, + startDate=None, + endDate=None, + startTime=datetime.time(0,0,0), + endTime=datetime.time(23,59,59), + walk=True, + timezone='ut', + all=0, + online=False, + ext=None): + + if ext==None: + ext = self.ext + self.timezone = timezone +# self.all = all +# self.online = online + self.path = path + + + if not(online): + #Busqueda de archivos offline + self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk) + else: + self.__searchFilesOnline(path, walk) + + if not(self.filenameList): + print "There is no files into the folder: %s"%(path) + sys.exit(-1) + +# self.__getExpParameters() + + self.fileIndex = -1 + + self.__setNextFileOffline() + + self.__readMetadata() + + self.blockIndex = 0 + + return + + def __searchFilesOffline(self, + path, + startDate, + endDate, + ext, + startTime=datetime.time(0,0,0), + endTime=datetime.time(23,59,59), + walk=True): + +# self.__setParameters(path, startDate, endDate, startTime, endTime, walk) +# +# self.__checkPath() +# +# self.__findDataForDates() +# +# self.__selectDataForTimes() +# +# for i in range(len(self.filenameList)): +# print "%s" %(self.filenameList[i]) + + pathList = [] + + if not walk: + #pathList.append(path) + multi_path = path.split(',') + for single_path in multi_path: + pathList.append(single_path) + + else: + #dirList = [] + multi_path = path.split(',') + for single_path in multi_path: + dirList = [] + for thisPath in os.listdir(single_path): + if not os.path.isdir(os.path.join(single_path,thisPath)): + continue + if not isDoyFolder(thisPath): + continue + + dirList.append(thisPath) + + if not(dirList): + return None, None + + thisDate = startDate + + while(thisDate <= endDate): + year = thisDate.timetuple().tm_year + doy = thisDate.timetuple().tm_yday + + matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*') + if len(matchlist) == 0: + thisDate += datetime.timedelta(1) + continue + for match in matchlist: + pathList.append(os.path.join(single_path,match)) + + thisDate += datetime.timedelta(1) + + if pathList == []: + print "Any folder was found for the date range: %s-%s" %(startDate, endDate) + return None, None + + print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate) + + filenameList = [] + datetimeList = [] + pathDict = {} + filenameList_to_sort = [] + + for i in range(len(pathList)): + + thisPath = pathList[i] + + fileList = glob.glob1(thisPath, "*%s" %ext) + fileList.sort() + pathDict.setdefault(fileList[0]) + pathDict[fileList[0]] = i + filenameList_to_sort.append(fileList[0]) + + filenameList_to_sort.sort() + + for file in filenameList_to_sort: + thisPath = pathList[pathDict[file]] + + fileList = glob.glob1(thisPath, "*%s" %ext) + fileList.sort() + + for file in fileList: + + filename = os.path.join(thisPath,file) + thisDatetime = self.__isFileinThisTime(filename, startTime, endTime) + + if not(thisDatetime): + continue + + filenameList.append(filename) + datetimeList.append(thisDatetime) + + if not(filenameList): + print "Any file was found for the time range %s - %s" %(startTime, endTime) + return None, None + + print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime) + print + + for i in range(len(filenameList)): + print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) + + self.filenameList = filenameList + self.datetimeList = datetimeList + + return pathList, filenameList + + def __isFileinThisTime(self, filename, startTime, endTime): + """ + Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. + + Inputs: + filename : nombre completo del archivo de datos en formato Jicamarca (.r) + + startTime : tiempo inicial del rango seleccionado en formato datetime.time + + endTime : tiempo final del rango seleccionado en formato datetime.time + + Return: + Boolean : Retorna True si el archivo de datos contiene datos en el rango de + fecha especificado, de lo contrario retorna False. + + Excepciones: + Si el archivo no existe o no puede ser abierto + Si la cabecera no puede ser leida. + + """ + + + try: + fp = fp = h5py.File(filename,'r') + except IOError: + traceback.print_exc() + raise IOError, "The file %s can't be opened" %(filename) + + grp = fp['Data'] + time = grp['time'] + time0 = time[:][0] + + fp.close() + + thisDatetime = datetime.datetime.utcfromtimestamp(time0) + + if self.timezone == 'lt': + thisDatetime = thisDatetime - datetime.timedelta(minutes = 300) + + thisTime = thisDatetime.time() + + if not ((startTime <= thisTime) and (endTime > thisTime)): + return None + + return thisDatetime + + def __checkPath(self): + if os.path.exists(self.path): + self.status = 1 + else: + self.status = 0 + print 'Path:%s does not exists'%self.path + + return + + def __setNextFileOffline(self): + idFile = self.fileIndex + idFile += 1 + + if not(idFile < len(self.filenameList)): + self.flagNoMoreFiles = 1 + print "No more Files" + return 0 + + filename = self.filenameList[idFile] + + filePointer = h5py.File(filename,'r') + + self.flagIsNewFile = 1 + self.fileIndex = idFile + self.filename = filename + + self.fp = filePointer + + print "Setting the file: %s"%self.filename + + self.__readMetadata() + + return 1 + + def __readMetadata(self): + grp = self.fp['Data'] + self.pathMeta = os.path.join(self.path, grp.attrs['metadata']) + filePointer = h5py.File(self.pathMeta,'r') + groupPointer = filePointer['Metadata'] + + listMetaname = [] + listMetadata = [] + for item in groupPointer.items(): + name = item[0] + + if name=='data shape': + self.nSamples = 1 + self.nPoints = 1 + self.nChannels = 1 + else: + data = groupPointer[name][:] + listMetaname.append(name) + listMetadata.append(data) + + if name=='type': + self.__initDataOut(name) + + filePointer.close() + + self.listMetadata = listMetaname + self.listMetadata = listMetadata + + return + + def __initDataOut(self, type): + + if 'type'=='Parameters': + self.dataOut = Parameters() + elif 'type'=='Spectra': + self.dataOut = Spectra() + elif 'type'=='Voltage': + self.dataOut = Voltage() + elif 'type'=='Correlation': + self.dataOut = Correlation() + + return + + def __setDataOut(self): + listMetadata = self.listMetadata + listMetaname = self.listMetaname + listDataname = self.listDataname + listData = self.listData + + blockIndex = self.blockIndex + + for i in range(len(listMetadata)): + setattr(self.dataOut,listMetaname[i],listMetadata[i]) + + for j in range(len(listData)): + setattr(self.dataOut,listDataname[j][blockIndex,:],listData[j][blockIndex,:]) + + return + + def getData(self): + + if self.flagNoMoreFiles: + self.dataOut.flagNoData = True + print 'Process finished' + return 0 + + if self.__hasNotDataInBuffer(): + self.__setNextFile() + + + if self.datablock == None: # setear esta condicion cuando no hayan datos por leers + self.dataOut.flagNoData = True + return 0 + + self.__setDataOut() + self.dataOut.flagNoData = False + + self.blockIndex += 1 + + return self.dataOut.data + + def run(self, **kwargs): + + if not(self.isConfig): + self.setup(**kwargs) + self.setObjProperties() + self.isConfig = True + + self.getData() + + return + +class HDF5Writer(Operation): + + ext = ".hdf5" + + optchar = "D" + + metaoptchar = "M" + + metaFile = None + + path = None + + setFile = None + + fp = None + + grp = None + + ds = None + + firsttime = True + + #Configurations + + blocksPerFile = None + + blockIndex = None + + dataOut = None + + #Data Arrays + + dataList = None + + metadataList = None + + dataDim = None + + def __init__(self): + + Operation.__init__(self) + self.isConfig = False + return + + + def setup(self, dataOut, **kwargs): + + self.path = kwargs['path'] + + if kwargs.has_key('ext'): + self.ext = kwargs['ext'] + else: + self.blocksPerFile = 10 + + if kwargs.has_key('blocksPerFile'): + self.blocksPerFile = kwargs['blocksPerFile'] + else: + self.blocksPerFile = 10 + + self.dataOut = dataOut + + self.metadataList = ['inputUnit','abscissaRange','heightRange'] + + self.dataList = ['data_param', 'data_error', 'data_SNR'] + + self.dataDim = numpy.zeros((len(self.dataList),3)) + + for i in range(len(self.dataList)): + + dataDim = getattr(self.dataOut, self.dataList[i]).shape + + if len(dataDim) == 3: + self.dataDim[i,:] = numpy.array(dataDim) + else: + self.dataDim[i,:-1] = numpy.array(dataDim) + self.dataDim[i,-1] = numpy.nan + + self.blockIndex = 0 + + return + + def putMetadata(self): + + fp = self.createMetadataFile() + self.writeMetadata(fp) + fp.close() + return + + def createMetadataFile(self): + ext = self.ext + path = self.path + setFile = self.setFile + + timeTuple = time.localtime(self.dataOut.utctime) + subfolder = '' + + fullpath = os.path.join( path, subfolder ) + if not( os.path.exists(fullpath) ): + os.mkdir(fullpath) + setFile = -1 #inicializo mi contador de seteo + else: + filesList = os.listdir( fullpath ) + if len( filesList ) > 0: + filesList = sorted( filesList, key=str.lower ) + filen = filesList[-1] + # el filename debera tener el siguiente formato + # 0 1234 567 89A BCDE (hex) + # x YYYY DDD SSS .ext + if isNumber( filen[8:11] ): + setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file + else: + setFile = -1 + else: + setFile = -1 #inicializo mi contador de seteo + + setFile += 1 + + file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar, + timeTuple.tm_year, + timeTuple.tm_yday, + setFile, + ext ) + + filename = os.path.join( path, subfolder, file ) + self.metaFile = file + #Setting HDF5 File + fp = h5py.File(filename,'w') + + return fp + + def writeMetadata(self, fp): + + grp = fp.create_group("Metadata") + + for i in range(len(self.metadataList)): + grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i])) + return + + def setNextFile(self): + + ext = self.ext + path = self.path + setFile = self.setFile + + if self.fp != None: + self.fp.close() + + timeTuple = time.localtime(self.dataOut.utctime) + subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) + + fullpath = os.path.join( path, subfolder ) + if not( os.path.exists(fullpath) ): + os.mkdir(fullpath) + setFile = -1 #inicializo mi contador de seteo + else: + filesList = os.listdir( fullpath ) + if len( filesList ) > 0: + filesList = sorted( filesList, key=str.lower ) + filen = filesList[-1] + # el filename debera tener el siguiente formato + # 0 1234 567 89A BCDE (hex) + # x YYYY DDD SSS .ext + if isNumber( filen[8:11] ): + setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file + else: + setFile = -1 + else: + setFile = -1 #inicializo mi contador de seteo + + setFile += 1 + + file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, + timeTuple.tm_year, + timeTuple.tm_yday, + setFile, + ext ) + + filename = os.path.join( path, subfolder, file ) + + #Setting HDF5 File + fp = h5py.File(filename,'w') + grp = fp.create_group("Data") + grp.attrs['metadata'] = self.metaFile + + + + ds = [] + data = [] + + for i in range(len(self.dataList)): + + grp0 = grp.create_group(self.dataList[i]) + + for j in range(int(self.dataDim[i,0])): + tableName = "channel" + str(j) + + if not(numpy.isnan(self.dataDim[i,2])): + ds0 = grp0.create_dataset(tableName, (1,1,1) , chunks = True) + else: + ds0 = grp0.create_dataset(tableName, (1,1) , chunks = True) + + ds.append(ds0) + data.append([]) + + ds0 = grp.create_dataset("time", (1,) , chunks = True) + ds.append(ds0) + data.append([]) + + #Saving variables + print 'Writing the file: %s'%filename + self.fp = fp + self.grp = grp + self.ds = ds + self.data = data + + self.setFile = setFile + self.firsttime = True + self.blockIndex = 0 + return + + def putData(self): + self.setBlock() + self.writeBlock() + + if self.blockIndex == self.blocksPerFile: + self.setNextFile() + return + + def setBlock(self): + + #Creating Arrays + data = self.data + ind = 0 + for i in range(len(self.dataList)): + dataAux = getattr(self.dataOut,self.dataList[i]) + + for j in range(int(self.dataDim[i,0])): + data[ind] = dataAux[j,:] + if not(numpy.isnan(self.dataDim[i,2])): + data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1)) + if not self.firsttime: + data[ind] = numpy.dstack((self.ds[ind][:], data[ind])) + else: + data[ind] = data[ind].reshape((1,data[ind].shape[0])) + if not self.firsttime: + data[ind] = numpy.vstack((self.ds[ind][:], data[ind])) + ind += 1 + + data[ind] = numpy.array([self.dataOut.utctime]) + if not self.firsttime: + self.data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind])) + self.data = data + + return + + def writeBlock(self): + + for i in range(len(self.ds)): + self.ds[i].shape = self.data[i].shape + self.ds[i][:] = self.data[i] + + self.blockIndex += 1 + + self.grp['blocksPerFile'] = self.blockIndex + + self.firsttime = False + return + + def run(self, dataOut, **kwargs): + if not(self.isConfig): + self.setup(dataOut, **kwargs) + self.isConfig = True + self.putMetadata() + self.setNextFile() + + self.putData() + return + diff --git a/schainpy/model/io/jrodataIO.py b/schainpy/model/io/jrodataIO.py index 43a7263..e9c4a8c 100644 --- a/schainpy/model/io/jrodataIO.py +++ b/schainpy/model/io/jrodataIO.py @@ -1,4 +1,5 @@ from jroIO_voltage import * from jroIO_spectra import * from jroIO_heispectra import * -from jroIO_amisr import * \ No newline at end of file +from jroIO_amisr import * +from jroIO_HDF5 import * \ No newline at end of file diff --git a/schainpy/model/proc/jroproc_parameters.py b/schainpy/model/proc/jroproc_parameters.py index 6e9f59d..ad54b2d 100644 --- a/schainpy/model/proc/jroproc_parameters.py +++ b/schainpy/model/proc/jroproc_parameters.py @@ -115,7 +115,7 @@ class ParametersProc(ProcessingUnit): self.dataOut.abscissaRange = self.dataIn.getLagTRange(1) self.dataOut.noise = self.dataIn.noise self.dataOut.normFactor = self.dataIn.normFactor - self.dataOut.SNR = self.dataIn.SNR + self.dataOut.data_SNR = self.dataIn.SNR self.dataOut.groupList = self.dataIn.pairsList self.dataOut.flagNoData = False @@ -138,7 +138,7 @@ class ParametersProc(ProcessingUnit): Affected: self.dataOut.data_param - self.dataOut.SNR + self.dataOut.data_SNR ''' data = self.dataOut.data_pre @@ -155,7 +155,7 @@ class ParametersProc(ProcessingUnit): data_param[ind,:,:] = self.__calculateMoments(data[ind,:,:], absc, noise[ind]) self.dataOut.data_param = data_param[:,1:,:] - self.dataOut.SNR = data_param[:,0] + self.dataOut.data_SNR = data_param[:,0] return def __calculateMoments(self, oldspec, oldfreq, n0, nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None): @@ -241,7 +241,7 @@ class ParametersProc(ProcessingUnit): self.dataOut.abscissaRange self.dataOut.noise self.dataOut.normFactor - self.dataOut.SNR + self.dataOut.data_SNR self.dataOut.groupList self.dataOut.nChannels @@ -254,7 +254,7 @@ class ParametersProc(ProcessingUnit): nHeights = self.dataOut.nHeights absc = self.dataOut.abscissaRange[:-1] noise = self.dataOut.noise - SNR = self.dataOut.SNR + SNR = self.dataOut.data_SNR pairsList = self.dataOut.groupList nChannels = self.dataOut.nChannels pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairsList, nChannels) @@ -1155,7 +1155,7 @@ class ParametersProc(ProcessingUnit): listChannels = groupArray.reshape((groupArray.size)) listChannels.sort() noise = self.dataIn.getNoise() - self.dataOut.SNR = self.__getSNR(self.dataIn.data_spc[listChannels,:,:], noise[listChannels]) + self.dataOut.data_SNR = self.__getSNR(self.dataIn.data_spc[listChannels,:,:], noise[listChannels]) for i in range(nGroups): coord = groupArray[i,:] @@ -1201,22 +1201,27 @@ class ParametersProc(ProcessingUnit): #Initial values data_spc = self.dataIn.data_spc[coord,:,h] - p0 = self.dataOut.library.initialValuesFunction(data_spc, constants) + p0 = numpy.array(self.dataOut.library.initialValuesFunction(data_spc, constants)) - #Least Squares - minp,covp,infodict,mesg,ier = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants),full_output=True) -# minp,covp = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants)) - #Chi square error - error0 = numpy.sum(infodict['fvec']**2)/(2*N) -# error0 = 0 - #Error with Jacobian - error1 = self.dataOut.library.errorFunction(minp,constants,LT) + try: + #Least Squares + minp,covp,infodict,mesg,ier = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants),full_output=True) +# minp,covp = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants)) + #Chi square error + error0 = numpy.sum(infodict['fvec']**2)/(2*N) + #Error with Jacobian + error1 = self.dataOut.library.errorFunction(minp,constants,LT) + except: + minp = p0*numpy.nan + error0 = numpy.nan + error1 = p0*numpy.nan + #Save if self.dataOut.data_param == None: - self.dataOut.data_param = numpy.zeros((nGroups, minp.size, nHeights))*numpy.nan - self.dataOut.error = numpy.zeros((nGroups, error1.size + 1, nHeights))*numpy.nan + self.dataOut.data_param = numpy.zeros((nGroups, p0.size, nHeights))*numpy.nan + self.dataOut.data_error = numpy.zeros((nGroups, p0.size + 1, nHeights))*numpy.nan - self.dataOut.error[i,:,h] = numpy.hstack((error0,error1)) + self.dataOut.data_error[i,:,h] = numpy.hstack((error0,error1)) self.dataOut.data_param[i,:,h] = minp return @@ -1569,7 +1574,7 @@ class WindProfiler(Operation): absc = dataOut.abscissaRange[:-1] noise = dataOut.noise heightRange = dataOut.getHeiRange() - SNR = dataOut.SNR + SNR = dataOut.data_SNR if technique == 'DBS': @@ -1597,7 +1602,7 @@ class WindProfiler(Operation): theta_y = theta_y[arrayChannel] velRadial0 = param[:,1,:] #Radial velocity - dataOut.data_output, dataOut.heightRange, dataOut.SNR = self.techniqueDBS(velRadial0, theta_x, theta_y, azimuth, correctFactor, horizontalOnly, heightRange, SNR) #DBS Function + dataOut.data_output, dataOut.heightRange, dataOut.data_SNR = self.techniqueDBS(velRadial0, theta_x, theta_y, azimuth, correctFactor, horizontalOnly, heightRange, SNR) #DBS Function elif technique == 'SA': @@ -1715,7 +1720,7 @@ class EWDriftsEstimation(Operation): def run(self, dataOut, zenith, zenithCorrection): heiRang = dataOut.heightList velRadial = dataOut.data_param[:,3,:] - SNR = dataOut.SNR + SNR = dataOut.data_SNR zenith = numpy.array(zenith) zenith -= zenithCorrection @@ -1736,7 +1741,7 @@ class EWDriftsEstimation(Operation): dataOut.heightList = heiRang1 dataOut.data_output = winds - dataOut.SNR = SNR1 + dataOut.data_SNR = SNR1 dataOut.initUtcTime = dataOut.ltctime dataOut.outputInterval = dataOut.timeInterval diff --git a/schainpy/test/EWDrifts_estimation01.py b/schainpy/test/EWDrifts_estimation01.py index b46e637..dbb9449 100644 --- a/schainpy/test/EWDrifts_estimation01.py +++ b/schainpy/test/EWDrifts_estimation01.py @@ -23,14 +23,15 @@ pathFigure = '/home/propietario/workspace/Graficos/drifts' path = "/home/soporte/Data/drifts" pathFigure = '/home/soporte/workspace/Graficos/drifts/prueba' +pathFile = '/home/soporte/Data/drifts/HDF5' -xmin = 11.75 -xmax = 14.75 +xmin = 0 +xmax = 24 #------------------------------------------------------------------------------------------------ readUnitConfObj = controllerObj.addReadUnit(datatype='VoltageReader', path=path, - startDate='2012/01/01', - endDate='2012/12/31', + startDate='2012/09/06', + endDate='2012/09/06', startTime='00:00:00', endTime='23:59:59', online=0, @@ -106,6 +107,10 @@ opObj11.addParameter(name='fit', value='1', format='int')#1--True/include fit opObj11.addParameter(name='save', value='1', format='bool') opObj11.addParameter(name='figpath', value = pathFigure, format='str') +opObj12 = procUnitConfObj2.addOperation(name='HDF5Writer', optype='other') +opObj12.addParameter(name='path', value=pathFile) +opObj12.addParameter(name='blocksPerFile', value='3', format='int') + opObj11 = procUnitConfObj2.addOperation(name='EWDriftsEstimation', optype='other') opObj11.addParameter(name='zenith', value='-3.80208,3.10658', format='floatlist') opObj11.addParameter(name='zenithCorrection', value='0.183201', format='float') diff --git a/schainpy/test/EWDrifts_estimation02.py b/schainpy/test/EWDrifts_estimation02.py new file mode 100644 index 0000000..8980f9d --- /dev/null +++ b/schainpy/test/EWDrifts_estimation02.py @@ -0,0 +1,65 @@ +# DIAS 19 Y 20 FEB 2014 +# Comprobacion de Resultados DBS con SA + +import os, sys + +path = os.path.split(os.getcwd())[0] +sys.path.append(path) + +from controller import * + +desc = "DBS Experiment Test" +filename = "DBStest.xml" + +controllerObj = Project() + +controllerObj.setup(id = '191', name='test01', description=desc) + +#Experimentos + +path = "/home/soporte/Data/drifts/HDF5" +pathFigure = '/home/soporte/workspace/Graficos/drifts/prueba' +pathFile = '/home/soporte/Data/drifts/HDF5' + +xmin = 0 +xmax = 24 +#------------------------------------------------------------------------------------------------ +readUnitConfObj = controllerObj.addReadUnit(datatype='HDF5Reader', + path=path, + startDate='2012/09/06', + endDate='2012/09/06', + startTime='00:00:00', + endTime='23:59:59', + timezone='lt', + walk=1) + +procUnitConfObj0 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId()) +#-------------------------------------------------------------------------------------------------- + +opObj11 = procUnitConfObj0.addOperation(name='EWDriftsEstimation', optype='other') +opObj11.addParameter(name='zenith', value='-3.80208,3.10658', format='floatlist') +opObj11.addParameter(name='zenithCorrection', value='0.183201', format='float') + +opObj23 = procUnitConfObj0.addOperation(name='EWDriftsPlot', optype='other') +opObj23.addParameter(name='id', value='4', format='int') +opObj23.addParameter(name='wintitle', value='EW Drifts', format='str') +opObj23.addParameter(name='save', value='1', format='bool') +opObj23.addParameter(name='figpath', value = pathFigure, format='str') +opObj23.addParameter(name='zminZonal', value='-150', format='int') +opObj23.addParameter(name='zmaxZonal', value='150', format='int') +opObj23.addParameter(name='zminVertical', value='-30', format='float') +opObj23.addParameter(name='zmaxVertical', value='30', format='float') +opObj23.addParameter(name='SNR_1', value='1', format='bool') +opObj23.addParameter(name='SNRmax', value='5', format='int') +# opObj23.addParameter(name='SNRthresh', value='-50', format='float') +opObj23.addParameter(name='xmin', value=xmin, format='float') +opObj23.addParameter(name='xmax', value=xmax, format='float') +#-------------------------------------------------------------------------------------------------- +print "Escribiendo el archivo XML" +controllerObj.writeXml(filename) +print "Leyendo el archivo XML" +controllerObj.readXml(filename) + +controllerObj.createObjects() +controllerObj.connectObjects() +controllerObj.run() \ No newline at end of file diff --git a/schainpy/test/WindProfiler_DBS01.py b/schainpy/test/WindProfiler_DBS01.py index 21843e9..fcd3b44 100644 --- a/schainpy/test/WindProfiler_DBS01.py +++ b/schainpy/test/WindProfiler_DBS01.py @@ -109,7 +109,7 @@ opObj21.addParameter(name='figpath', value=pathFigure, format='str') opObj21.addParameter(name='zmin', value='5', format='int') opObj21.addParameter(name='zmax', value='90', format='int') -opObj21 = procUnitConfObj2.addOperation(name='RadialVelocityPlot', optype='other') +opObj21 = procUnitConfObj2.addOperation(name='ParametersPlot', optype='other') opObj21.addParameter(name='id', value='5', format='int') opObj21.addParameter(name='wintitle', value='Radial Velocity Plot', format='str') opObj21.addParameter(name='save', value='1', format='bool') @@ -119,6 +119,20 @@ opObj21.addParameter(name='SNRmax', value='60', format='int') opObj21.addParameter(name='SNRthresh', value='0', format='float') opObj21.addParameter(name='xmin', value=xmin, format='float') opObj21.addParameter(name='xmax', value=xmax, format='float') + +opObj21 = procUnitConfObj2.addOperation(name='ParametersPlot', optype='other') +opObj21.addParameter(name='id', value='6', format='int') +opObj21.addParameter(name='wintitle', value='Spectral width Plot', format='str') +opObj21.addParameter(name='save', value='1', format='bool') +opObj21.addParameter(name='figpath', value=pathFigure, format='str') +opObj21.addParameter(name='SNRmin', value='-10', format='int') +opObj21.addParameter(name='SNRmax', value='60', format='int') +opObj21.addParameter(name='SNRthresh', value='0', format='float') +opObj21.addParameter(name='xmin', value=xmin, format='float') +opObj21.addParameter(name='xmax', value=xmax, format='float') +opObj21.addParameter(name='zmin', value=0, format='float') +opObj21.addParameter(name='paramIndex', value=2, format='int') +opObj21.addParameter(name='onlyPositive', value=1, format='bool') opObj22 = procUnitConfObj2.addOperation(name='WindProfiler', optype='other') opObj22.addParameter(name='technique', value='DBS', format='str') diff --git a/schainpy/test/WindProfiler_SA01.py b/schainpy/test/WindProfiler_SA01.py index 493ecc0..8d0a8c6 100644 --- a/schainpy/test/WindProfiler_SA01.py +++ b/schainpy/test/WindProfiler_SA01.py @@ -20,7 +20,7 @@ controllerObj.setup(id = '191', name='test01', description=desc) #2014050 19 Feb 2014 path = '/home/soporte/Data/MST/SA/d2014050' -pathFigure = '/home/soporte/workspace/Graficos/SA/new1/' +pathFigure = '/home/soporte/workspace/Graficos/SA/prueba1/' xmin = '15.5' xmax = '24' startTime = '15:30:00'