jroIO_amisr.py
691 lines
| 24.2 KiB
| text/x-python
|
PythonLexer
Daniel Valdez
|
r491 | ''' | ||
| @author: Daniel Suarez | ||||
| ''' | ||||
| import os | ||||
| import sys | ||||
| import glob | ||||
| import fnmatch | ||||
| import datetime | ||||
|
Daniel Valdez
|
r497 | import time | ||
|
Daniel Valdez
|
r491 | import re | ||
| import h5py | ||||
| import numpy | ||||
Miguel Valdez
|
r568 | from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation | ||
| from schainpy.model.data.jroamisr import AMISR | ||||
|
Daniel Valdez
|
r491 | |||
|
Miguel Valdez
|
r587 | try: | ||
| from gevent import sleep | ||||
| except: | ||||
| from time import sleep | ||||
|
Daniel Valdez
|
r491 | class RadacHeader(): | ||
| def __init__(self, fp): | ||||
| header = 'Raw11/Data/RadacHeader' | ||||
Percy Condor
|
r1383 | self.beamCodeByPulse = fp.get(header + '/BeamCode') | ||
|
Daniel Valdez
|
r491 | self.beamCode = fp.get('Raw11/Data/Beamcodes') | ||
|
Percy Condor
|
r1383 | self.code = fp.get(header + '/Code') | ||
| self.frameCount = fp.get(header + '/FrameCount') | ||||
| self.modeGroup = fp.get(header + '/ModeGroup') | ||||
| self.nsamplesPulse = fp.get(header + '/NSamplesPulse') | ||||
| self.pulseCount = fp.get(header + '/PulseCount') | ||||
| self.radacTime = fp.get(header + '/RadacTime') | ||||
| self.timeCount = fp.get(header + '/TimeCount') | ||||
| self.timeStatus = fp.get(header + '/TimeStatus') | ||||
| self.nrecords = self.pulseCount.shape[0] # nblocks | ||||
| self.npulses = self.pulseCount.shape[1] # nprofile | ||||
| self.nsamples = self.nsamplesPulse[0, 0] # ngates | ||||
|
Daniel Valdez
|
r491 | self.nbeams = self.beamCode.shape[1] | ||
|
Daniel Valdez
|
r497 | |||
|
Daniel Valdez
|
r491 | |||
| def getIndexRangeToPulse(self, idrecord=0): | ||||
|
Percy Condor
|
r1383 | # indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0) | ||
| # startPulseCountId = indexToZero[0][0] | ||||
| # endPulseCountId = startPulseCountId - 1 | ||||
| # range1 = numpy.arange(startPulseCountId,self.npulses,1) | ||||
| # range2 = numpy.arange(0,startPulseCountId,1) | ||||
| # return range1, range2 | ||||
|
Daniel Valdez
|
r508 | zero = 0 | ||
|
Percy Condor
|
r1383 | npulse = max(self.pulseCount[0, :] + 1) - 1 | ||
| looking_index = numpy.where(self.pulseCount.value[idrecord, :] == npulse)[0] | ||||
|
Daniel Valdez
|
r508 | getLastIndex = looking_index[-1] | ||
|
Percy Condor
|
r1383 | index_data = numpy.arange(0, getLastIndex + 1, 1) | ||
| index_buffer = numpy.arange(getLastIndex + 1, self.npulses, 1) | ||||
|
Daniel Valdez
|
r497 | return index_data, index_buffer | ||
|
Daniel Valdez
|
r491 | |||
| class AMISRReader(ProcessingUnit): | ||||
| path = None | ||||
| startDate = None | ||||
| endDate = None | ||||
| startTime = None | ||||
| endTime = None | ||||
| walk = None | ||||
| isConfig = False | ||||
| def __init__(self): | ||||
| self.set = None | ||||
| self.subset = None | ||||
| self.extension_file = '.h5' | ||||
| self.dtc_str = 'dtc' | ||||
| self.dtc_id = 0 | ||||
| self.status = True | ||||
| self.isConfig = False | ||||
| self.dirnameList = [] | ||||
| self.filenameList = [] | ||||
| self.fileIndex = None | ||||
| self.flagNoMoreFiles = False | ||||
| self.flagIsNewFile = 0 | ||||
| self.filename = '' | ||||
| self.amisrFilePointer = None | ||||
| self.radacHeaderObj = None | ||||
| self.dataOut = self.__createObjByDefault() | ||||
| self.datablock = None | ||||
| self.rest_datablock = None | ||||
| self.range = None | ||||
| self.idrecord_count = 0 | ||||
| self.profileIndex = 0 | ||||
|
Daniel Valdez
|
r497 | self.index_amisr_sample = None | ||
| self.index_amisr_buffer = None | ||||
|
Daniel Valdez
|
r491 | self.beamCodeByFrame = None | ||
| self.radacTimeByFrame = None | ||||
|
Percy Condor
|
r1383 | # atributos originales tal y como esta en el archivo de datos | ||
|
Daniel Valdez
|
r491 | self.beamCodesFromFile = None | ||
| self.radacTimeFromFile = None | ||||
| self.rangeFromFile = None | ||||
| self.dataByFrame = None | ||||
| self.dataset = None | ||||
| self.beamCodeDict = {} | ||||
| self.beamRangeDict = {} | ||||
|
Percy Condor
|
r1383 | # experiment cgf file | ||
|
Daniel Valdez
|
r491 | self.npulsesint_fromfile = None | ||
| self.recordsperfile_fromfile = None | ||||
| self.nbeamcodes_fromfile = None | ||||
| self.ngates_fromfile = None | ||||
| self.ippSeconds_fromfile = None | ||||
| self.frequency_h5file = None | ||||
| self.__firstFile = True | ||||
| self.buffer_radactime = None | ||||
|
Daniel Valdez
|
r497 | self.index4_schain_datablock = None | ||
| self.index4_buffer = None | ||||
| self.schain_datablock = None | ||||
| self.buffer = None | ||||
| self.linear_pulseCount = None | ||||
| self.npulseByFrame = None | ||||
| self.profileIndex_offset = None | ||||
| self.timezone = 'ut' | ||||
|
Daniel Valdez
|
r508 | |||
| self.__waitForNewFile = 20 | ||||
| self.__filename_online = None | ||||
|
Daniel Valdez
|
r491 | def __createObjByDefault(self): | ||
| dataObj = AMISR() | ||||
| return dataObj | ||||
|
Percy Condor
|
r1383 | def __setParameters(self, path='', startDate='', endDate='', startTime='', endTime='', walk=''): | ||
|
Daniel Valdez
|
r491 | self.path = path | ||
| self.startDate = startDate | ||||
| self.endDate = endDate | ||||
| self.startTime = startTime | ||||
| self.endTime = endTime | ||||
| self.walk = walk | ||||
| def __checkPath(self): | ||||
| if os.path.exists(self.path): | ||||
| self.status = 1 | ||||
| else: | ||||
| self.status = 0 | ||||
|
Percy Condor
|
r1383 | print('Path:%s does not exists' % self.path) | ||
|
Daniel Valdez
|
r491 | |||
| return | ||||
| def __selDates(self, amisr_dirname_format): | ||||
| try: | ||||
| year = int(amisr_dirname_format[0:4]) | ||||
| month = int(amisr_dirname_format[4:6]) | ||||
| dom = int(amisr_dirname_format[6:8]) | ||||
|
Percy Condor
|
r1383 | thisDate = datetime.date(year, month, dom) | ||
|
Daniel Valdez
|
r491 | |||
|
Percy Condor
|
r1383 | if (thisDate >= self.startDate and thisDate <= self.endDate): | ||
|
Daniel Valdez
|
r491 | return amisr_dirname_format | ||
| except: | ||||
| return None | ||||
|
Percy Condor
|
r1383 | def __findDataForDates(self, online=False): | ||
|
Daniel Valdez
|
r491 | |||
| if not(self.status): | ||||
| return None | ||||
| pat = '\d+.\d+' | ||||
|
Percy Condor
|
r1383 | dirnameList = [re.search(pat, x) for x in os.listdir(self.path)] | ||
| dirnameList = [x for x in dirnameList if x != None] | ||||
|
Daniel Valdez
|
r491 | dirnameList = [x.string for x in dirnameList] | ||
|
Daniel Valdez
|
r508 | if not(online): | ||
| dirnameList = [self.__selDates(x) for x in dirnameList] | ||||
|
Percy Condor
|
r1383 | dirnameList = [x for x in dirnameList if x != None] | ||
| if len(dirnameList) > 0: | ||||
|
Daniel Valdez
|
r491 | self.status = 1 | ||
| self.dirnameList = dirnameList | ||||
| self.dirnameList.sort() | ||||
| else: | ||||
| self.status = 0 | ||||
| return None | ||||
| def __getTimeFromData(self): | ||||
|
Percy Condor
|
r1383 | startDateTime_Reader = datetime.datetime.combine(self.startDate, self.startTime) | ||
| endDateTime_Reader = datetime.datetime.combine(self.endDate, self.endTime) | ||||
|
Daniel Valdez
|
r499 | |||
|
Percy Condor
|
r1383 | print('Filtering Files from %s to %s' % (startDateTime_Reader, endDateTime_Reader)) | ||
George Yong
|
r1167 | print('........................................') | ||
|
Daniel Valdez
|
r499 | filter_filenameList = [] | ||
|
Daniel Valdez
|
r504 | self.filenameList.sort() | ||
|
Percy Condor
|
r1383 | for i in range(len(self.filenameList) - 1): | ||
|
Daniel Valdez
|
r500 | filename = self.filenameList[i] | ||
|
Percy Condor
|
r1383 | fp = h5py.File(filename, 'r') | ||
|
Daniel Valdez
|
r499 | time_str = fp.get('Time/RadacTimeString') | ||
| startDateTimeStr_File = time_str[0][0].split('.')[0] | ||||
| junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S') | ||||
|
Percy Condor
|
r1383 | startDateTime_File = datetime.datetime(junk.tm_year, junk.tm_mon, junk.tm_mday, junk.tm_hour, junk.tm_min, junk.tm_sec) | ||
|
Daniel Valdez
|
r499 | |||
| endDateTimeStr_File = time_str[-1][-1].split('.')[0] | ||||
| junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S') | ||||
|
Percy Condor
|
r1383 | endDateTime_File = datetime.datetime(junk.tm_year, junk.tm_mon, junk.tm_mday, junk.tm_hour, junk.tm_min, junk.tm_sec) | ||
|
Daniel Valdez
|
r499 | |||
| fp.close() | ||||
| if self.timezone == 'lt': | ||||
|
Percy Condor
|
r1383 | startDateTime_File = startDateTime_File - datetime.timedelta(minutes=300) | ||
| endDateTime_File = endDateTime_File - datetime.timedelta(minutes=300) | ||||
|
Daniel Valdez
|
r499 | |||
|
Percy Condor
|
r1383 | if (endDateTime_File >= startDateTime_Reader and endDateTime_File < endDateTime_Reader): | ||
| # self.filenameList.remove(filename) | ||||
|
Daniel Valdez
|
r499 | filter_filenameList.append(filename) | ||
| filter_filenameList.sort() | ||||
| self.filenameList = filter_filenameList | ||||
| return 1 | ||||
|
Daniel Valdez
|
r491 | |||
| def __filterByGlob1(self, dirName): | ||||
|
Percy Condor
|
r1383 | filter_files = glob.glob1(dirName, '*.*%s' % self.extension_file) | ||
|
Daniel Valdez
|
r491 | filterDict = {} | ||
| filterDict.setdefault(dirName) | ||||
| filterDict[dirName] = filter_files | ||||
| return filterDict | ||||
| def __getFilenameList(self, fileListInKeys, dirList): | ||||
| for value in fileListInKeys: | ||||
|
George Yong
|
r1167 | dirName = list(value.keys())[0] | ||
|
Daniel Valdez
|
r491 | for file in value[dirName]: | ||
| filename = os.path.join(dirName, file) | ||||
| self.filenameList.append(filename) | ||||
|
Daniel Valdez
|
r508 | def __selectDataForTimes(self, online=False): | ||
|
Percy Condor
|
r1383 | # aun no esta implementado el filtro for tiempo | ||
|
Daniel Valdez
|
r491 | if not(self.status): | ||
| return None | ||||
|
Percy Condor
|
r1383 | dirList = [os.path.join(self.path, x) for x in self.dirnameList] | ||
|
Daniel Valdez
|
r491 | |||
| fileListInKeys = [self.__filterByGlob1(x) for x in dirList] | ||||
| self.__getFilenameList(fileListInKeys, dirList) | ||||
|
Daniel Valdez
|
r508 | if not(online): | ||
|
Percy Condor
|
r1383 | # filtro por tiempo | ||
|
Daniel Valdez
|
r508 | if not(self.all): | ||
| self.__getTimeFromData() | ||||
|
Daniel Valdez
|
r499 | |||
|
Percy Condor
|
r1383 | if len(self.filenameList) > 0: | ||
|
Daniel Valdez
|
r508 | self.status = 1 | ||
| self.filenameList.sort() | ||||
| else: | ||||
| self.status = 0 | ||||
| return None | ||||
|
Daniel Valdez
|
r510 | |||
|
Daniel Valdez
|
r491 | else: | ||
|
Percy Condor
|
r1383 | # get the last file - 1 | ||
|
Daniel Valdez
|
r508 | self.filenameList = [self.filenameList[-2]] | ||
|
Daniel Valdez
|
r510 | |||
| new_dirnameList = [] | ||||
| for dirname in self.dirnameList: | ||||
| junk = numpy.array([dirname in x for x in self.filenameList]) | ||||
| junk_sum = junk.sum() | ||||
| if junk_sum > 0: | ||||
| new_dirnameList.append(dirname) | ||||
| self.dirnameList = new_dirnameList | ||||
| return 1 | ||||
|
Daniel Valdez
|
r508 | |||
Juan C. Espinoza
|
r1052 | def searchFilesOnLine(self, | ||
|
Daniel Valdez
|
r508 | path, | ||
| walk=True): | ||||
| startDate = datetime.datetime.utcnow().date() | ||||
| endDate = datetime.datetime.utcnow().date() | ||||
| self.__setParameters(path=path, startDate=startDate, endDate=endDate, walk=walk) | ||||
| self.__checkPath() | ||||
| self.__findDataForDates(online=True) | ||||
| self.dirnameList = [self.dirnameList[-1]] | ||||
| self.__selectDataForTimes(online=True) | ||||
| return | ||||
|
Daniel Valdez
|
r491 | |||
|
Juan C. Espinoza
|
r1052 | def searchFilesOffLine(self, | ||
|
Daniel Valdez
|
r491 | path, | ||
| startDate, | ||||
| endDate, | ||||
|
Percy Condor
|
r1383 | startTime=datetime.time(0, 0, 0), | ||
| endTime=datetime.time(23, 59, 59), | ||||
|
Daniel Valdez
|
r491 | walk=True): | ||
| self.__setParameters(path, startDate, endDate, startTime, endTime, walk) | ||||
| self.__checkPath() | ||||
| self.__findDataForDates() | ||||
| self.__selectDataForTimes() | ||||
| for i in range(len(self.filenameList)): | ||||
|
Percy Condor
|
r1383 | print("%s" % (self.filenameList[i])) | ||
|
Daniel Valdez
|
r491 | |||
| return | ||||
| def __setNextFileOffline(self): | ||||
| idFile = self.fileIndex | ||||
| while (True): | ||||
| idFile += 1 | ||||
| if not(idFile < len(self.filenameList)): | ||||
| self.flagNoMoreFiles = 1 | ||||
|
George Yong
|
r1167 | print("No more Files") | ||
|
Daniel Valdez
|
r491 | return 0 | ||
| filename = self.filenameList[idFile] | ||||
|
Percy Condor
|
r1383 | amisrFilePointer = h5py.File(filename, 'r') | ||
|
Daniel Valdez
|
r491 | |||
| break | ||||
| self.flagIsNewFile = 1 | ||||
| self.fileIndex = idFile | ||||
| self.filename = filename | ||||
| self.amisrFilePointer = amisrFilePointer | ||||
|
Percy Condor
|
r1383 | print("Setting the file: %s" % self.filename) | ||
|
Daniel Valdez
|
r491 | |||
| return 1 | ||||
|
Daniel Valdez
|
r508 | |||
| def __setNextFileOnline(self): | ||||
| filename = self.filenameList[0] | ||||
| if self.__filename_online != None: | ||||
| self.__selectDataForTimes(online=True) | ||||
| filename = self.filenameList[0] | ||||
| while self.__filename_online == filename: | ||||
|
Percy Condor
|
r1383 | print('waiting %d seconds to get a new file...' % (self.__waitForNewFile)) | ||
|
Miguel Valdez
|
r587 | sleep(self.__waitForNewFile) | ||
|
Daniel Valdez
|
r508 | self.__selectDataForTimes(online=True) | ||
| filename = self.filenameList[0] | ||||
| self.__filename_online = filename | ||||
|
Percy Condor
|
r1383 | self.amisrFilePointer = h5py.File(filename, 'r') | ||
|
Daniel Valdez
|
r508 | self.flagIsNewFile = 1 | ||
| self.filename = filename | ||||
|
Percy Condor
|
r1383 | print("Setting the file: %s" % self.filename) | ||
|
Daniel Valdez
|
r508 | return 1 | ||
|
Daniel Valdez
|
r491 | def __readHeader(self): | ||
| self.radacHeaderObj = RadacHeader(self.amisrFilePointer) | ||||
|
Percy Condor
|
r1383 | # update values from experiment cfg file | ||
|
Daniel Valdez
|
r491 | if self.radacHeaderObj.nrecords == self.recordsperfile_fromfile: | ||
| self.radacHeaderObj.nrecords = self.recordsperfile_fromfile | ||||
| self.radacHeaderObj.nbeams = self.nbeamcodes_fromfile | ||||
| self.radacHeaderObj.npulses = self.npulsesint_fromfile | ||||
| self.radacHeaderObj.nsamples = self.ngates_fromfile | ||||
|
Percy Condor
|
r1383 | # looking index list for data | ||
| start_index = self.radacHeaderObj.pulseCount[0, :][0] | ||||
|
Daniel Valdez
|
r497 | end_index = self.radacHeaderObj.npulses | ||
|
George Yong
|
r1167 | range4data = list(range(start_index, end_index)) | ||
|
Daniel Valdez
|
r497 | self.index4_schain_datablock = numpy.array(range4data) | ||
| buffer_start_index = 0 | ||||
|
Percy Condor
|
r1383 | buffer_end_index = self.radacHeaderObj.pulseCount[0, :][0] | ||
|
George Yong
|
r1167 | range4buffer = list(range(buffer_start_index, buffer_end_index)) | ||
|
Daniel Valdez
|
r497 | self.index4_buffer = numpy.array(range4buffer) | ||
| self.linear_pulseCount = numpy.array(range4data + range4buffer) | ||||
|
Percy Condor
|
r1383 | self.npulseByFrame = max(self.radacHeaderObj.pulseCount[0, :] + 1) | ||
|
Daniel Valdez
|
r497 | |||
|
Percy Condor
|
r1383 | # get tuning frequency | ||
| frequency_h5file_dataset = self.amisrFilePointer.get('Rx' + '/TuningFrequency') | ||||
| self.frequency_h5file = frequency_h5file_dataset[0, 0] | ||||
|
Daniel Valdez
|
r491 | |||
| self.flagIsNewFile = 1 | ||||
| def __getBeamCode(self): | ||||
| self.beamCodeDict = {} | ||||
| self.beamRangeDict = {} | ||||
|
Daniel Valdez
|
r499 | beamCodeMap = self.amisrFilePointer.get('Setup/BeamcodeMap') | ||
|
Percy Condor
|
r1383 | for i in range(len(self.radacHeaderObj.beamCode[0, :])): | ||
|
Daniel Valdez
|
r491 | self.beamCodeDict.setdefault(i) | ||
| self.beamRangeDict.setdefault(i) | ||||
|
Percy Condor
|
r1383 | beamcodeValue = self.radacHeaderObj.beamCode[0, i] | ||
| beamcodeIndex = numpy.where(beamCodeMap[:, 0] == beamcodeValue)[0][0] | ||||
|
Daniel Valdez
|
r499 | x = beamCodeMap[beamcodeIndex][1] | ||
| y = beamCodeMap[beamcodeIndex][2] | ||||
| z = beamCodeMap[beamcodeIndex][3] | ||||
| self.beamCodeDict[i] = [beamcodeValue, x, y, z] | ||||
|
Daniel Valdez
|
r491 | |||
|
Percy Condor
|
r1383 | just4record0 = self.radacHeaderObj.beamCodeByPulse[0, :] | ||
|
Daniel Valdez
|
r491 | |||
|
George Yong
|
r1167 | for i in range(len(list(self.beamCodeDict.values()))): | ||
|
Percy Condor
|
r1383 | xx = numpy.where(just4record0 == list(self.beamCodeDict.values())[i][0]) | ||
|
Daniel Valdez
|
r504 | indexPulseByBeam = self.linear_pulseCount[xx[0]] | ||
| self.beamRangeDict[i] = indexPulseByBeam | ||||
|
Daniel Valdez
|
r491 | |||
| def __getExpParameters(self): | ||||
| if not(self.status): | ||||
| return None | ||||
| experimentCfgPath = os.path.join(self.path, self.dirnameList[0], 'Setup') | ||||
|
Percy Condor
|
r1383 | expFinder = glob.glob1(experimentCfgPath, '*.exp') | ||
| if len(expFinder) == 0: | ||||
|
Daniel Valdez
|
r491 | self.status = 0 | ||
| return None | ||||
|
Percy Condor
|
r1383 | experimentFilename = os.path.join(experimentCfgPath, expFinder[0]) | ||
|
Daniel Valdez
|
r491 | |||
| f = open(experimentFilename) | ||||
| lines = f.readlines() | ||||
| f.close() | ||||
|
Percy Condor
|
r1383 | parmsList = ['npulsesint*', 'recordsperfile*', 'nbeamcodes*', 'ngates*'] | ||
|
Daniel Valdez
|
r491 | filterList = [fnmatch.filter(lines, x) for x in parmsList] | ||
|
Percy Condor
|
r1383 | values = [re.sub(r'\D', "", x[0]) for x in filterList] | ||
|
Daniel Valdez
|
r491 | |||
| self.npulsesint_fromfile = int(values[0]) | ||||
| self.recordsperfile_fromfile = int(values[1]) | ||||
| self.nbeamcodes_fromfile = int(values[2]) | ||||
| self.ngates_fromfile = int(values[3]) | ||||
| tufileFinder = fnmatch.filter(lines, 'tufile=*') | ||||
| tufile = tufileFinder[0].split('=')[1].split('\n')[0] | ||||
|
Daniel Valdez
|
r499 | tufile = tufile.split('\r')[0] | ||
|
Percy Condor
|
r1383 | tufilename = os.path.join(experimentCfgPath, tufile) | ||
|
Daniel Valdez
|
r491 | |||
| f = open(tufilename) | ||||
| lines = f.readlines() | ||||
| f.close() | ||||
|
Percy Condor
|
r1383 | self.ippSeconds_fromfile = float(lines[1].split()[2]) / 1E6 | ||
|
Daniel Valdez
|
r491 | |||
| self.status = 1 | ||||
| def __setIdsAndArrays(self): | ||||
| self.dataByFrame = self.__setDataByFrame() | ||||
| self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[0, :] | ||||
| self.readRanges() | ||||
|
Daniel Valdez
|
r497 | self.index_amisr_sample, self.index_amisr_buffer = self.radacHeaderObj.getIndexRangeToPulse(0) | ||
|
Daniel Valdez
|
r491 | self.radacTimeByFrame = numpy.zeros(self.radacHeaderObj.npulses) | ||
|
Daniel Valdez
|
r508 | if len(self.index_amisr_buffer) > 0: | ||
| self.buffer_radactime = numpy.zeros_like(self.radacTimeByFrame) | ||||
|
Daniel Valdez
|
r491 | |||
|
Percy Condor
|
r1383 | def __setNextFile(self, online=False): | ||
|
Daniel Valdez
|
r491 | |||
|
Daniel Valdez
|
r508 | if not(online): | ||
| newFile = self.__setNextFileOffline() | ||||
| else: | ||||
| newFile = self.__setNextFileOnline() | ||||
|
Daniel Valdez
|
r491 | |||
| if not(newFile): | ||||
| return 0 | ||||
| self.__readHeader() | ||||
| if self.__firstFile: | ||||
| self.__setIdsAndArrays() | ||||
| self.__firstFile = False | ||||
| self.__getBeamCode() | ||||
| self.readDataBlock() | ||||
|
Percy Condor
|
r1383 | def setup(self, path=None, | ||
| startDate=None, | ||||
| endDate=None, | ||||
| startTime=datetime.time(0, 0, 0), | ||||
| endTime=datetime.time(23, 59, 59), | ||||
|
Daniel Valdez
|
r497 | walk=True, | ||
|
Daniel Valdez
|
r499 | timezone='ut', | ||
|
Daniel Valdez
|
r508 | all=0, | ||
| online=False): | ||||
|
Daniel Valdez
|
r491 | |||
|
Daniel Valdez
|
r497 | self.timezone = timezone | ||
|
Daniel Valdez
|
r499 | self.all = all | ||
|
Daniel Valdez
|
r508 | self.online = online | ||
| if not(online): | ||||
|
Percy Condor
|
r1383 | # Busqueda de archivos offline | ||
|
Juan C. Espinoza
|
r1052 | self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk) | ||
|
Daniel Valdez
|
r508 | else: | ||
|
Juan C. Espinoza
|
r1052 | self.searchFilesOnLine(path, walk) | ||
|
Daniel Valdez
|
r491 | |||
| if not(self.filenameList): | ||||
|
Percy Condor
|
r1383 | print("There is no files into the folder: %s" % (path)) | ||
|
Daniel Valdez
|
r491 | |||
| sys.exit(-1) | ||||
| self.__getExpParameters() | ||||
| self.fileIndex = -1 | ||||
|
Daniel Valdez
|
r508 | self.__setNextFile(online) | ||
|
Daniel Valdez
|
r497 | |||
|
Daniel Valdez
|
r504 | # first_beamcode = self.radacHeaderObj.beamCodeByPulse[0,0] | ||
| # index = numpy.where(self.radacHeaderObj.beamCodeByPulse[0,:]!=first_beamcode)[0][0] | ||||
|
Percy Condor
|
r1383 | self.profileIndex_offset = self.radacHeaderObj.pulseCount[0, :][0] | ||
|
Daniel Valdez
|
r497 | self.profileIndex = self.profileIndex_offset | ||
|
Daniel Valdez
|
r491 | |||
| def readRanges(self): | ||||
| dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range') | ||||
|
Daniel Valdez
|
r504 | |||
|
Percy Condor
|
r1383 | self.rangeFromFile = numpy.reshape(dataset.value, (-1)) | ||
|
Daniel Valdez
|
r504 | return self.rangeFromFile | ||
|
Daniel Valdez
|
r491 | |||
|
Percy Condor
|
r1383 | def readRadacTime(self, idrecord, range1, range2): | ||
|
Daniel Valdez
|
r491 | self.radacTimeFromFile = self.radacHeaderObj.radacTime.value | ||
| radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses)) | ||||
|
Percy Condor
|
r1383 | # radacTimeByFrame = dataset[idrecord - 1,range1] | ||
| # radacTimeByFrame = dataset[idrecord,range2] | ||||
|
Daniel Valdez
|
r491 | |||
| return radacTimeByFrame | ||||
| def readBeamCode(self, idrecord, range1, range2): | ||||
| dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode') | ||||
| beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses)) | ||||
| self.beamCodesFromFile = dataset.value | ||||
|
Percy Condor
|
r1383 | # beamcodeByFrame[range1] = dataset[idrecord - 1, range1] | ||
| # beamcodeByFrame[range2] = dataset[idrecord, range2] | ||||
|
Daniel Valdez
|
r491 | beamcodeByFrame[range1] = dataset[idrecord, range1] | ||
| beamcodeByFrame[range2] = dataset[idrecord, range2] | ||||
| return beamcodeByFrame | ||||
| def __setDataByFrame(self): | ||||
|
Percy Condor
|
r1383 | ndata = 2 # porque es complejo | ||
|
Daniel Valdez
|
r491 | dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata)) | ||
| return dataByFrame | ||||
| def __readDataSet(self): | ||||
| dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data') | ||||
| return dataset | ||||
| def __setDataBlock(self,): | ||||
|
Percy Condor
|
r1383 | real = self.dataByFrame[:, :, 0] # asumo que 0 es real | ||
| imag = self.dataByFrame[:, :, 1] # asumo que 1 es imaginario | ||||
| datablock = real + imag * 1j # armo el complejo | ||||
|
Daniel Valdez
|
r491 | return datablock | ||
|
Percy Condor
|
r1383 | def readSamples_version1(self, idrecord): | ||
| # estas tres primeras lineas solo se deben ejecutar una vez | ||||
|
Daniel Valdez
|
r491 | if self.flagIsNewFile: | ||
|
Percy Condor
|
r1383 | # reading dataset | ||
|
Daniel Valdez
|
r491 | self.dataset = self.__readDataSet() | ||
| self.flagIsNewFile = 0 | ||||
| if idrecord == 0: | ||||
|
Percy Condor
|
r1383 | self.dataByFrame[self.index4_schain_datablock, : , :] = self.dataset[0, self.index_amisr_sample, :, :] | ||
|
Daniel Valdez
|
r497 | self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[0, self.index_amisr_sample] | ||
|
Daniel Valdez
|
r491 | datablock = self.__setDataBlock() | ||
|
Daniel Valdez
|
r508 | if len(self.index_amisr_buffer) > 0: | ||
|
Percy Condor
|
r1383 | self.buffer = self.dataset[0, self.index_amisr_buffer, :, :] | ||
|
Daniel Valdez
|
r508 | self.buffer_radactime = self.radacHeaderObj.radacTime[0, self.index_amisr_buffer] | ||
|
Daniel Valdez
|
r497 | |||
|
Daniel Valdez
|
r491 | return datablock | ||
|
Daniel Valdez
|
r508 | if len(self.index_amisr_buffer) > 0: | ||
|
Percy Condor
|
r1383 | self.dataByFrame[self.index4_buffer, :, :] = self.buffer.copy() | ||
|
Daniel Valdez
|
r508 | self.radacTimeByFrame[self.index4_buffer] = self.buffer_radactime.copy() | ||
|
Percy Condor
|
r1383 | self.dataByFrame[self.index4_schain_datablock, :, :] = self.dataset[idrecord, self.index_amisr_sample, :, :] | ||
|
Daniel Valdez
|
r497 | self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_sample] | ||
| datablock = self.__setDataBlock() | ||||
|
Daniel Valdez
|
r508 | if len(self.index_amisr_buffer) > 0: | ||
| self.buffer = self.dataset[idrecord, self.index_amisr_buffer, :, :] | ||||
| self.buffer_radactime = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_buffer] | ||||
|
Daniel Valdez
|
r491 | |||
| return datablock | ||||
|
Percy Condor
|
r1383 | def readSamples(self, idrecord): | ||
|
Daniel Valdez
|
r491 | if self.flagIsNewFile: | ||
| self.dataByFrame = self.__setDataByFrame() | ||||
| self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :] | ||||
|
Percy Condor
|
r1383 | # reading ranges | ||
|
Daniel Valdez
|
r491 | self.readRanges() | ||
|
Percy Condor
|
r1383 | # reading dataset | ||
|
Daniel Valdez
|
r491 | self.dataset = self.__readDataSet() | ||
| self.flagIsNewFile = 0 | ||||
| self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :] | ||||
| self.dataByFrame = self.dataset[idrecord, :, :, :] | ||||
| datablock = self.__setDataBlock() | ||||
| return datablock | ||||
| def readDataBlock(self): | ||||
| self.datablock = self.readSamples_version1(self.idrecord_count) | ||||
|
Percy Condor
|
r1383 | # self.datablock = self.readSamples(self.idrecord_count) | ||
| # print 'record:', self.idrecord_count | ||||
|
Daniel Valdez
|
r491 | |||
| self.idrecord_count += 1 | ||||
| self.profileIndex = 0 | ||||
| if self.idrecord_count >= self.radacHeaderObj.nrecords: | ||||
| self.idrecord_count = 0 | ||||
| self.flagIsNewFile = 1 | ||||
| def readNextBlock(self): | ||||
| self.readDataBlock() | ||||
| if self.flagIsNewFile: | ||||
|
Daniel Valdez
|
r508 | self.__setNextFile(self.online) | ||
|
Daniel Valdez
|
r491 | pass | ||
| def __hasNotDataInBuffer(self): | ||||
|
Percy Condor
|
r1383 | # self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record | ||
|
Daniel Valdez
|
r491 | if self.profileIndex >= self.radacHeaderObj.npulses: | ||
| return 1 | ||||
| return 0 | ||||
| def printUTC(self): | ||||
|
George Yong
|
r1167 | print(self.dataOut.utctime) | ||
| print('') | ||||
|
Daniel Valdez
|
r491 | |||
| def setObjProperties(self): | ||||
|
Miguel Valdez
|
r527 | |||
|
Percy Condor
|
r1383 | self.dataOut.heightList = self.rangeFromFile / 1000.0 # km | ||
|
Daniel Valdez
|
r491 | self.dataOut.nProfiles = self.radacHeaderObj.npulses | ||
| self.dataOut.nRecords = self.radacHeaderObj.nrecords | ||||
| self.dataOut.nBeams = self.radacHeaderObj.nbeams | ||||
| self.dataOut.ippSeconds = self.ippSeconds_fromfile | ||||
|
Miguel Valdez
|
r527 | # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt | ||
|
Daniel Valdez
|
r491 | self.dataOut.frequency = self.frequency_h5file | ||
|
Daniel Valdez
|
r497 | self.dataOut.npulseByFrame = self.npulseByFrame | ||
|
Daniel Valdez
|
r491 | self.dataOut.nBaud = None | ||
| self.dataOut.nCode = None | ||||
| self.dataOut.code = None | ||||
| self.dataOut.beamCodeDict = self.beamCodeDict | ||||
| self.dataOut.beamRangeDict = self.beamRangeDict | ||||
|
Daniel Valdez
|
r497 | |||
| if self.timezone == 'lt': | ||||
|
Percy Condor
|
r1383 | self.dataOut.timeZone = time.timezone / 60. # get the timezone in minutes | ||
|
Daniel Valdez
|
r497 | else: | ||
|
Percy Condor
|
r1383 | self.dataOut.timeZone = 0 # by default time is UTC | ||
|
Daniel Valdez
|
r497 | |||
|
Daniel Valdez
|
r491 | def getData(self): | ||
| if self.flagNoMoreFiles: | ||||
| self.dataOut.flagNoData = True | ||||
| return 0 | ||||
| if self.__hasNotDataInBuffer(): | ||||
| self.readNextBlock() | ||||
|
Percy Condor
|
r1383 | if self.datablock is None: # setear esta condicion cuando no hayan datos por leers | ||
|
Daniel Valdez
|
r491 | self.dataOut.flagNoData = True | ||
| return 0 | ||||
|
Percy Condor
|
r1383 | self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex, :], (1, -1)) | ||
|
Daniel Valdez
|
r491 | |||
| self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex] | ||||
|
Daniel Valdez
|
r501 | self.dataOut.profileIndex = self.profileIndex | ||
|
Daniel Valdez
|
r491 | self.dataOut.flagNoData = False | ||
| self.profileIndex += 1 | ||||
| return self.dataOut.data | ||||
| def run(self, **kwargs): | ||||
| if not(self.isConfig): | ||||
| self.setup(**kwargs) | ||||
| self.setObjProperties() | ||||
| self.isConfig = True | ||||
|
Percy Condor
|
r1383 | self.getData() | ||