##// END OF EJS Templates
The RTI for Spectra-1d Plot shows the normalized power.
The RTI for Spectra-1d Plot shows the normalized power.

File last commit:

r497:85140003652c
r498:09578ea8c29c
Show More
jroIO_amisr.py
573 lines | 19.3 KiB | text/x-python | PythonLexer
'''
@author: Daniel Suarez
'''
import os
import sys
import glob
import fnmatch
import datetime
import time
import re
import h5py
import numpy
from model.proc.jroproc_base import ProcessingUnit, Operation
from model.data.jroamisr import AMISR
class RadacHeader():
def __init__(self, fp):
header = 'Raw11/Data/RadacHeader'
self.beamCodeByPulse = fp.get(header+'/BeamCode')
self.beamCode = fp.get('Raw11/Data/Beamcodes')
self.code = fp.get(header+'/Code')
self.frameCount = fp.get(header+'/FrameCount')
self.modeGroup = fp.get(header+'/ModeGroup')
self.nsamplesPulse = fp.get(header+'/NSamplesPulse')
self.pulseCount = fp.get(header+'/PulseCount')
self.radacTime = fp.get(header+'/RadacTime')
self.timeCount = fp.get(header+'/TimeCount')
self.timeStatus = fp.get(header+'/TimeStatus')
self.nrecords = self.pulseCount.shape[0] #nblocks
self.npulses = self.pulseCount.shape[1] #nprofile
self.nsamples = self.nsamplesPulse[0,0] #ngates
self.nbeams = self.beamCode.shape[1]
def getIndexRangeToPulse(self, idrecord=0):
#indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
#startPulseCountId = indexToZero[0][0]
#endPulseCountId = startPulseCountId - 1
#range1 = numpy.arange(startPulseCountId,self.npulses,1)
#range2 = numpy.arange(0,startPulseCountId,1)
#return range1, range2
looking_zeros_index = numpy.where(self.pulseCount.value[idrecord,:]==0)[0]
getLastIndexZero = looking_zeros_index[-1]
index_data = numpy.arange(0,getLastIndexZero,1)
index_buffer = numpy.arange(getLastIndexZero,self.npulses,1)
return index_data, index_buffer
class AMISRReader(ProcessingUnit):
path = None
startDate = None
endDate = None
startTime = None
endTime = None
walk = None
isConfig = False
def __init__(self):
self.set = None
self.subset = None
self.extension_file = '.h5'
self.dtc_str = 'dtc'
self.dtc_id = 0
self.status = True
self.isConfig = False
self.dirnameList = []
self.filenameList = []
self.fileIndex = None
self.flagNoMoreFiles = False
self.flagIsNewFile = 0
self.filename = ''
self.amisrFilePointer = None
self.radacHeaderObj = None
self.dataOut = self.__createObjByDefault()
self.datablock = None
self.rest_datablock = None
self.range = None
self.idrecord_count = 0
self.profileIndex = 0
self.index_amisr_sample = None
self.index_amisr_buffer = None
self.beamCodeByFrame = None
self.radacTimeByFrame = None
#atributos originales tal y como esta en el archivo de datos
self.beamCodesFromFile = None
self.radacTimeFromFile = None
self.rangeFromFile = None
self.dataByFrame = None
self.dataset = None
self.beamCodeDict = {}
self.beamRangeDict = {}
#experiment cgf file
self.npulsesint_fromfile = None
self.recordsperfile_fromfile = None
self.nbeamcodes_fromfile = None
self.ngates_fromfile = None
self.ippSeconds_fromfile = None
self.frequency_h5file = None
self.__firstFile = True
self.buffer_radactime = None
self.index4_schain_datablock = None
self.index4_buffer = None
self.schain_datablock = None
self.buffer = None
self.linear_pulseCount = None
self.npulseByFrame = None
self.profileIndex_offset = None
self.timezone = 'ut'
def __createObjByDefault(self):
dataObj = AMISR()
return dataObj
def __setParameters(self,path,startDate,endDate,startTime,endTime,walk):
self.path = path
self.startDate = startDate
self.endDate = endDate
self.startTime = startTime
self.endTime = endTime
self.walk = walk
def __checkPath(self):
if os.path.exists(self.path):
self.status = 1
else:
self.status = 0
print 'Path:%s does not exists'%self.path
return
def __selDates(self, amisr_dirname_format):
try:
year = int(amisr_dirname_format[0:4])
month = int(amisr_dirname_format[4:6])
dom = int(amisr_dirname_format[6:8])
thisDate = datetime.date(year,month,dom)
if (thisDate>=self.startDate and thisDate <= self.endDate):
return amisr_dirname_format
except:
return None
def __findDataForDates(self):
if not(self.status):
return None
pat = '\d+.\d+'
dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
dirnameList = filter(lambda x:x!=None,dirnameList)
dirnameList = [x.string for x in dirnameList]
dirnameList = [self.__selDates(x) for x in dirnameList]
dirnameList = filter(lambda x:x!=None,dirnameList)
if len(dirnameList)>0:
self.status = 1
self.dirnameList = dirnameList
self.dirnameList.sort()
else:
self.status = 0
return None
def __getTimeFromData(self):
pass
def __filterByGlob1(self, dirName):
filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
filterDict = {}
filterDict.setdefault(dirName)
filterDict[dirName] = filter_files
return filterDict
def __getFilenameList(self, fileListInKeys, dirList):
for value in fileListInKeys:
dirName = value.keys()[0]
for file in value[dirName]:
filename = os.path.join(dirName, file)
self.filenameList.append(filename)
def __selectDataForTimes(self):
#aun no esta implementado el filtro for tiempo
if not(self.status):
return None
dirList = [os.path.join(self.path,x) for x in self.dirnameList]
fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
self.__getFilenameList(fileListInKeys, dirList)
if len(self.filenameList)>0:
self.status = 1
self.filenameList.sort()
else:
self.status = 0
return None
def __searchFilesOffline(self,
path,
startDate,
endDate,
startTime=datetime.time(0,0,0),
endTime=datetime.time(23,59,59),
walk=True):
self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
self.__checkPath()
self.__findDataForDates()
self.__selectDataForTimes()
for i in range(len(self.filenameList)):
print "%s" %(self.filenameList[i])
return
def __setNextFileOffline(self):
idFile = self.fileIndex
while (True):
idFile += 1
if not(idFile < len(self.filenameList)):
self.flagNoMoreFiles = 1
print "No more Files"
return 0
filename = self.filenameList[idFile]
amisrFilePointer = h5py.File(filename,'r')
break
self.flagIsNewFile = 1
self.fileIndex = idFile
self.filename = filename
self.amisrFilePointer = amisrFilePointer
print "Setting the file: %s"%self.filename
return 1
def __readHeader(self):
self.radacHeaderObj = RadacHeader(self.amisrFilePointer)
#update values from experiment cfg file
if self.radacHeaderObj.nrecords == self.recordsperfile_fromfile:
self.radacHeaderObj.nrecords = self.recordsperfile_fromfile
self.radacHeaderObj.nbeams = self.nbeamcodes_fromfile
self.radacHeaderObj.npulses = self.npulsesint_fromfile
self.radacHeaderObj.nsamples = self.ngates_fromfile
#looking index list for data
start_index = self.radacHeaderObj.pulseCount[0,:][0]
end_index = self.radacHeaderObj.npulses
range4data = range(start_index, end_index)
self.index4_schain_datablock = numpy.array(range4data)
buffer_start_index = 0
buffer_end_index = self.radacHeaderObj.pulseCount[0,:][0]
range4buffer = range(buffer_start_index, buffer_end_index)
self.index4_buffer = numpy.array(range4buffer)
self.linear_pulseCount = numpy.array(range4data + range4buffer)
self.npulseByFrame = max(self.radacHeaderObj.pulseCount[0,:]+1)
#get tuning frequency
frequency_h5file_dataset = self.amisrFilePointer.get('Rx'+'/TuningFrequency')
self.frequency_h5file = frequency_h5file_dataset[0,0]
self.flagIsNewFile = 1
def __getBeamCode(self):
self.beamCodeDict = {}
self.beamRangeDict = {}
for i in range(len(self.radacHeaderObj.beamCode[0,:])):
self.beamCodeDict.setdefault(i)
self.beamRangeDict.setdefault(i)
self.beamCodeDict[i] = self.radacHeaderObj.beamCode[0,i]
just4record0 = self.radacHeaderObj.beamCodeByPulse[0,:]
for i in range(len(self.beamCodeDict.values())):
xx = numpy.where(just4record0==self.beamCodeDict.values()[i])
self.beamRangeDict[i] = xx[0]
def __getExpParameters(self):
if not(self.status):
return None
experimentCfgPath = os.path.join(self.path, self.dirnameList[0], 'Setup')
expFinder = glob.glob1(experimentCfgPath,'*.exp')
if len(expFinder)== 0:
self.status = 0
return None
experimentFilename = os.path.join(experimentCfgPath,expFinder[0])
f = open(experimentFilename)
lines = f.readlines()
f.close()
parmsList = ['npulsesint*','recordsperfile*','nbeamcodes*','ngates*']
filterList = [fnmatch.filter(lines, x) for x in parmsList]
values = [re.sub(r'\D',"",x[0]) for x in filterList]
self.npulsesint_fromfile = int(values[0])
self.recordsperfile_fromfile = int(values[1])
self.nbeamcodes_fromfile = int(values[2])
self.ngates_fromfile = int(values[3])
tufileFinder = fnmatch.filter(lines, 'tufile=*')
tufile = tufileFinder[0].split('=')[1].split('\n')[0]
tufilename = os.path.join(experimentCfgPath,tufile)
f = open(tufilename)
lines = f.readlines()
f.close()
self.ippSeconds_fromfile = float(lines[1].split()[2])/1E6
self.status = 1
def __setIdsAndArrays(self):
self.dataByFrame = self.__setDataByFrame()
self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[0, :]
self.readRanges()
self.index_amisr_sample, self.index_amisr_buffer = self.radacHeaderObj.getIndexRangeToPulse(0)
self.radacTimeByFrame = numpy.zeros(self.radacHeaderObj.npulses)
self.buffer_radactime = numpy.zeros_like(self.radacTimeByFrame)
def __setNextFile(self):
newFile = self.__setNextFileOffline()
if not(newFile):
return 0
self.__readHeader()
if self.__firstFile:
self.__setIdsAndArrays()
self.__firstFile = False
self.__getBeamCode()
self.readDataBlock()
def setup(self,path=None,
startDate=None,
endDate=None,
startTime=datetime.time(0,0,0),
endTime=datetime.time(23,59,59),
walk=True,
timezone='ut',):
self.timezone = timezone
#Busqueda de archivos offline
self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
if not(self.filenameList):
print "There is no files into the folder: %s"%(path)
sys.exit(-1)
self.__getExpParameters()
self.fileIndex = -1
self.__setNextFile()
first_beamcode = self.radacHeaderObj.beamCodeByPulse[0,0]
index = numpy.where(self.radacHeaderObj.beamCodeByPulse[0,:]!=first_beamcode)[0][0]
self.profileIndex_offset = self.radacHeaderObj.pulseCount[0,:][index]
self.profileIndex = self.profileIndex_offset
def readRanges(self):
dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range')
#self.rangeFromFile = dataset.value
self.rangeFromFile = numpy.reshape(dataset.value,(-1))
return range
def readRadacTime(self,idrecord, range1, range2):
self.radacTimeFromFile = self.radacHeaderObj.radacTime.value
radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
#radacTimeByFrame = dataset[idrecord - 1,range1]
#radacTimeByFrame = dataset[idrecord,range2]
return radacTimeByFrame
def readBeamCode(self, idrecord, range1, range2):
dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode')
beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
self.beamCodesFromFile = dataset.value
#beamcodeByFrame[range1] = dataset[idrecord - 1, range1]
#beamcodeByFrame[range2] = dataset[idrecord, range2]
beamcodeByFrame[range1] = dataset[idrecord, range1]
beamcodeByFrame[range2] = dataset[idrecord, range2]
return beamcodeByFrame
def __setDataByFrame(self):
ndata = 2 # porque es complejo
dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata))
return dataByFrame
def __readDataSet(self):
dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
return dataset
def __setDataBlock(self,):
real = self.dataByFrame[:,:,0] #asumo que 0 es real
imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario
datablock = real + imag*1j #armo el complejo
return datablock
def readSamples_version1(self,idrecord):
#estas tres primeras lineas solo se deben ejecutar una vez
if self.flagIsNewFile:
#reading dataset
self.dataset = self.__readDataSet()
self.flagIsNewFile = 0
if idrecord == 0:
self.dataByFrame[self.index4_schain_datablock, : ,:] = self.dataset[0, self.index_amisr_sample,:,:]
self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[0, self.index_amisr_sample]
datablock = self.__setDataBlock()
self.buffer = self.dataset[0, self.index_amisr_buffer,:,:]
self.buffer_radactime = self.radacHeaderObj.radacTime[0, self.index_amisr_buffer]
return datablock
self.dataByFrame[self.index4_buffer,:,:] = self.buffer.copy()
self.radacTimeByFrame[self.index4_buffer] = self.buffer_radactime.copy()
self.dataByFrame[self.index4_schain_datablock,:,:] = self.dataset[idrecord, self.index_amisr_sample,:,:]
self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_sample]
datablock = self.__setDataBlock()
self.buffer = self.dataset[idrecord, self.index_amisr_buffer, :, :]
self.buffer_radactime = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_buffer]
return datablock
def readSamples(self,idrecord):
if self.flagIsNewFile:
self.dataByFrame = self.__setDataByFrame()
self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :]
#reading ranges
self.readRanges()
#reading dataset
self.dataset = self.__readDataSet()
self.flagIsNewFile = 0
self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :]
self.dataByFrame = self.dataset[idrecord, :, :, :]
datablock = self.__setDataBlock()
return datablock
def readDataBlock(self):
self.datablock = self.readSamples_version1(self.idrecord_count)
#self.datablock = self.readSamples(self.idrecord_count)
#print 'record:', self.idrecord_count
self.idrecord_count += 1
self.profileIndex = 0
if self.idrecord_count >= self.radacHeaderObj.nrecords:
self.idrecord_count = 0
self.flagIsNewFile = 1
def readNextBlock(self):
self.readDataBlock()
if self.flagIsNewFile:
self.__setNextFile()
pass
def __hasNotDataInBuffer(self):
#self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record
if self.profileIndex >= self.radacHeaderObj.npulses:
return 1
return 0
def printUTC(self):
print self.dataOut.utctime
print ''
def setObjProperties(self):
self.dataOut.heightList = self.rangeFromFile/1000.0 #km
self.dataOut.nProfiles = self.radacHeaderObj.npulses
self.dataOut.nRecords = self.radacHeaderObj.nrecords
self.dataOut.nBeams = self.radacHeaderObj.nbeams
self.dataOut.ippSeconds = self.ippSeconds_fromfile
self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
self.dataOut.frequency = self.frequency_h5file
self.dataOut.npulseByFrame = self.npulseByFrame
self.dataOut.nBaud = None
self.dataOut.nCode = None
self.dataOut.code = None
self.dataOut.beamCodeDict = self.beamCodeDict
self.dataOut.beamRangeDict = self.beamRangeDict
if self.timezone == 'lt':
self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
else:
self.dataOut.timeZone = 0 #by default time is UTC
def getData(self):
if self.flagNoMoreFiles:
self.dataOut.flagNoData = True
print 'Process finished'
return 0
if self.__hasNotDataInBuffer():
self.readNextBlock()
if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
self.dataOut.flagNoData = True
return 0
self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1))
self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex]
self.dataOut.flagNoData = False
self.profileIndex += 1
return self.dataOut.data
def run(self, **kwargs):
if not(self.isConfig):
self.setup(**kwargs)
self.setObjProperties()
self.isConfig = True
self.getData()