jroIO_matlab.py
273 lines
| 8.4 KiB
| text/x-python
|
PythonLexer
Alexander Valdez
|
r591 | from schainpy.model.data.jrodata import * | ||
| from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation | ||||
| from schainpy.model.io.jroIO_base import * | ||||
| import scipy.io as sio | ||||
| import pprint | ||||
| import numpy as np | ||||
| from os import listdir | ||||
| from os.path import isfile, join | ||||
| import datetime | ||||
| import cmath | ||||
|
Alexander Valdez
|
r593 | class matoffReader(ProcessingUnit): | ||
|
Alexander Valdez
|
r591 | |||
Percy Condor
|
r1383 | index = None | ||
| list = None | ||||
| firsttime = True | ||||
| utccounter = None | ||||
| utcfiletime = None | ||||
| utcmatcounter = 0 | ||||
| utcfirst = None | ||||
| utclist = None | ||||
|
Alexander Valdez
|
r591 | |||
| def __init__(self): | ||||
| self.dataOut = Spectra() | ||||
|
Alexander Valdez
|
r593 | self.online = False | ||
|
Alexander Valdez
|
r591 | return | ||
|
Alexander Valdez
|
r593 | def __setHeader(self, datastuff): | ||
|
Percy Condor
|
r1383 | self.dataOut.pairsList = [(0, 1)] | ||
| self.dataOut.channelList = list(range(np.array(datastuff.get('power')).shape[1])) | ||||
| self.dataOut.nProfiles = len(np.array(datastuff.get('vel')).flatten()) # this! | ||||
|
Alexander Valdez
|
r598 | self.dataOut.nIncohInt = 20 | ||
|
Percy Condor
|
r1383 | self.dataOut.nCohInt = 1 # this! | ||
| self.dataOut.ippSeconds = 0.004 # this! | ||||
|
Alexander Valdez
|
r598 | self.dataOut.nFFTPoints = len(np.array(datastuff.get('vel')).flatten()) | ||
|
Alexander Valdez
|
r593 | self.dataOut.timeZone = 0 | ||
| self.dataOut.heightList = np.array(datastuff.get('hts')).flatten() | ||||
|
Alexander Valdez
|
r591 | |||
|
Alexander Valdez
|
r593 | def __readFile(self, currentfile): | ||
George Yong
|
r1167 | print("Reading from this file:" + currentfile) | ||
|
Alexander Valdez
|
r591 | |||
|
Percy Condor
|
r1383 | # filesplit=currentfile.split("\\") | ||
| filesplit = currentfile.split("/") | ||||
| newsplit = filesplit[-2] | ||||
| newnewsplit = newsplit.split(".") | ||||
| newnewsplit = [int(i) for i in newnewsplit] | ||||
| gooblist = datetime.datetime(newnewsplit[0], newnewsplit[1], newnewsplit[2], newnewsplit[3], newnewsplit[4], newnewsplit[5]) | ||||
| self.utcfirst = (gooblist - datetime.datetime(1970, 1, 1)).total_seconds() | ||||
|
Alexander Valdez
|
r593 | |||
|
Percy Condor
|
r1383 | newsplit = filesplit[-1] | ||
| newnewsplit = newsplit.split(".") | ||||
| goobnum = newnewsplit[0] | ||||
| goobnum = int(goobnum) | ||||
|
Alexander Valdez
|
r593 | |||
|
Percy Condor
|
r1383 | self.utcfirst = self.utcfirst + goobnum * 2 | ||
|
Alexander Valdez
|
r593 | # if (currentfile[43:]=='0.mat'): | ||
| # self.utcmatcounter=0 | ||||
| # self.utcfirst=self.utclist[self.index] | ||||
| # if (self.utcmatcounter>60): | ||||
| # self.utcmatcounter=0 | ||||
| # print self.utcmatcounter | ||||
|
George Yong
|
r1167 | print(self.utcfirst) | ||
|
Alexander Valdez
|
r593 | try: | ||
|
Percy Condor
|
r1383 | datastuff = sio.loadmat(currentfile) | ||
|
Alexander Valdez
|
r593 | except: | ||
| return None, None | ||||
|
Percy Condor
|
r1383 | dataphase = datastuff.get('phase') | ||
| data3 = datastuff.get('doppler0') | ||||
| data4 = datastuff.get('doppler1') | ||||
| data3 = np.array(data3) | ||||
|
Alexander Valdez
|
r593 | data4 = np.array(data4) | ||
|
Percy Condor
|
r1383 | datacoh = datastuff.get('coherence2') | ||
|
Alexander Valdez
|
r593 | |||
|
Percy Condor
|
r1383 | datacohphase = datacoh * np.exp(-dataphase * 1j) | ||
|
Alexander Valdez
|
r593 | # data31 = np.fliplr(data3) | ||
| # data41 = np.fliplr(data4) | ||||
|
Percy Condor
|
r1383 | data31 = data3.reshape((1, data3.shape[0], data3.shape[1])) | ||
| data41 = data4.reshape((1, data4.shape[0], data4.shape[1])) | ||||
| datacohphase1 = datacohphase.reshape((1, datacoh.shape[0], datacoh.shape[1])) | ||||
|
Alexander Valdez
|
r593 | |||
|
Percy Condor
|
r1383 | datastack = np.vstack((data31, data41)) | ||
|
Alexander Valdez
|
r593 | |||
| self.__setHeader(datastuff) | ||||
| spc = datastack | ||||
| cspc = datacohphase1 | ||||
| return spc, cspc | ||||
|
Percy Condor
|
r1383 | def __findFiles(self, path, startDate=None, endDate=None, startTime=datetime.time(0, 0, 0), endTime=datetime.time(23, 59, 59)): | ||
|
Alexander Valdez
|
r591 | |||
|
Alexander Valdez
|
r593 | if startDate == None: | ||
|
Percy Condor
|
r1383 | startDate = datetime.date(1970, 1, 1) | ||
|
Alexander Valdez
|
r593 | |||
| if endDate == None: | ||||
|
Percy Condor
|
r1383 | endDate = datetime.date(2050, 1, 1) | ||
|
Alexander Valdez
|
r593 | |||
|
Percy Condor
|
r1383 | startsearch1 = datetime.datetime.combine(startDate, startTime) | ||
| startsearch2 = (startsearch1 - datetime.datetime(1970, 1, 1)).total_seconds() | ||||
| endsearch1 = datetime.datetime.combine(endDate, endTime) | ||||
| endsearch2 = (endsearch1 - datetime.datetime(1970, 1, 1)).total_seconds() | ||||
|
Alexander Valdez
|
r593 | |||
| dirList = listdir(path) | ||||
| dirList = sorted(dirList) | ||||
|
Percy Condor
|
r1383 | dirListFiltered = [] | ||
| fileListFiltered = [] | ||||
| utclist = [] | ||||
|
Alexander Valdez
|
r591 | |||
|
Alexander Valdez
|
r593 | if not dirList: | ||
|
George Yong
|
r1167 | print("No directories found") | ||
|
Alexander Valdez
|
r593 | return [] | ||
|
Percy Condor
|
r1383 | # if self.online: | ||
|
Alexander Valdez
|
r598 | # dirList= [dirList[-1]] | ||
| if self.online: | ||||
|
Alexander Valdez
|
r602 | currentdate = datetime.datetime.now() | ||
|
Percy Condor
|
r1383 | strsplit1 = currentdate.strftime('%Y.%m.%d') | ||
| dirList = fnmatch.filter(dirList, strsplit1 + '*') | ||||
|
Alexander Valdez
|
r598 | |||
|
Alexander Valdez
|
r593 | for thisDir in dirList: | ||
| if not os.path.isdir(os.path.join(path, thisDir)): | ||||
| continue | ||||
|
Percy Condor
|
r1383 | strsplit = thisDir.split('.') | ||
| timeints = [int(i) for i in strsplit] | ||||
| timelist = datetime.datetime(timeints[0], timeints[1], timeints[2], timeints[3], timeints[4], timeints[5]) | ||||
| utctime = (timelist - datetime.datetime(1970, 1, 1)).total_seconds() | ||||
|
Alexander Valdez
|
r591 | |||
|
Alexander Valdez
|
r593 | if not self.online: | ||
| if (utctime > endsearch2): | ||||
| continue | ||||
| if (utctime < startsearch2): | ||||
| continue | ||||
|
Alexander Valdez
|
r591 | |||
|
Alexander Valdez
|
r593 | dirListFiltered.append(thisDir) | ||
| utclist.append(utctime) | ||||
| if not dirListFiltered: | ||||
|
George Yong
|
r1167 | print("filtro") | ||
|
Alexander Valdez
|
r593 | return [] | ||
| for thisDir in dirListFiltered: | ||||
| pathFile = os.path.join(self.path, thisDir) | ||||
| fileList = os.listdir(pathFile) | ||||
| if not fileList: | ||||
| continue | ||||
|
Alexander Valdez
|
r591 | |||
|
Alexander Valdez
|
r593 | for k in range(len(fileList)): | ||
|
Percy Condor
|
r1383 | thisFile = str(k) + '.mat' | ||
|
Alexander Valdez
|
r593 | |||
| if not os.path.isfile(os.path.join(pathFile, thisFile)): | ||||
| continue | ||||
| fileListFiltered.append(os.path.join(pathFile, thisFile)) | ||||
| return fileListFiltered | ||||
|
Percy Condor
|
r1383 | def __getNextOnlineFile(self, seconds=40): | ||
|
Alexander Valdez
|
r593 | |||
| filename = self.__getNextOfflineFile() | ||||
| if filename: | ||||
| return filename | ||||
| ncurrentfiles = len(self.fileList) | ||||
| nTries = 0 | ||||
| while (True): | ||||
| filelist = self.__findFiles(self.path) | ||||
| if len(filelist) > ncurrentfiles: | ||||
| break | ||||
|
Alexander Valdez
|
r591 | |||
|
Alexander Valdez
|
r593 | nTries += 1 | ||
|
Alexander Valdez
|
r591 | |||
|
Alexander Valdez
|
r593 | if nTries > 3: | ||
| break | ||||
|
Alexander Valdez
|
r591 | |||
|
Percy Condor
|
r1383 | print("Waiting %d seconds ..." % seconds) | ||
|
Alexander Valdez
|
r598 | time.sleep(40) | ||
|
Alexander Valdez
|
r593 | |||
| if not (len(filelist) > ncurrentfiles): | ||||
| return None | ||||
|
Alexander Valdez
|
r591 | |||
|
Alexander Valdez
|
r593 | self.fileList = filelist | ||
| filename = self.__getNextOfflineFile() | ||||
|
Alexander Valdez
|
r591 | |||
|
Alexander Valdez
|
r593 | return filename | ||
| def __getNextOfflineFile(self): | ||||
| if self.index >= len(self.fileList): | ||||
| return None | ||||
|
Percy Condor
|
r1383 | filename = self.fileList[self.index] | ||
|
Alexander Valdez
|
r593 | self.index += 1 | ||
| return filename | ||||
| def __getNextFile(self): | ||||
|
Alexander Valdez
|
r591 | |||
|
Alexander Valdez
|
r593 | if self.online: | ||
| filename = self.__getNextOnlineFile() | ||||
| else: | ||||
| filename = self.__getNextOfflineFile() | ||||
| return filename | ||||
|
Alexander Valdez
|
r591 | |||
|
Percy Condor
|
r1383 | def setup(self, path, startDate=None, endDate=None, startTime=datetime.time(0, 0, 0), endTime=datetime.time(23, 59, 59)): | ||
|
Alexander Valdez
|
r593 | |||
| fileList = self.__findFiles(path, startDate, endDate, startTime, endTime) | ||||
| if self.online: | ||||
|
Percy Condor
|
r1383 | self.index = len(fileList) - 1 | ||
|
Alexander Valdez
|
r593 | else: | ||
| self.index = 0 | ||||
|
Alexander Valdez
|
r591 | |||
|
Alexander Valdez
|
r593 | self.fileList = fileList | ||
|
George Yong
|
r1167 | print("fin setup") | ||
|
Alexander Valdez
|
r593 | |||
|
Percy Condor
|
r1383 | def run(self, path=None, startDate=None, endDate=None, | ||
| startTime=datetime.time(0, 0, 0), | ||||
| endTime=datetime.time(23, 59, 59), | ||||
| walk=True, timezone='ut', | ||||
| all=0, online=False, ext=None, **kwargs): | ||||
| self.path = path | ||||
| self.ext = ext | ||||
| self.startDate = startDate | ||||
| self.endDate = endDate | ||||
| self.startTime = startTime | ||||
| self.endTime = endTime | ||||
|
Alexander Valdez
|
r593 | self.online = online | ||
| self.dataOut.flagNoData = True | ||||
|
Percy Condor
|
r1383 | if (self.firsttime == True): | ||
|
Alexander Valdez
|
r593 | self.setup(path, startDate, endDate, startTime, endTime) | ||
|
Percy Condor
|
r1383 | self.firsttime = False | ||
|
Alexander Valdez
|
r591 | |||
|
Alexander Valdez
|
r593 | |||
| if not self.fileList: | ||||
| self.dataOut.flagNoData = True | ||||
|
George Yong
|
r1167 | print("lista vacia") | ||
|
Alexander Valdez
|
r593 | return | ||
| currentfile = self.__getNextFile() | ||||
| if not currentfile: | ||||
| "no file" | ||||
| return | ||||
| spc, cspc = self.__readFile(currentfile) | ||||
|
Percy Condor
|
r1383 | if spc != None: | ||
|
Alexander Valdez
|
r591 | |||
|
Alexander Valdez
|
r593 | self.dataOut.data_spc = spc | ||
| self.dataOut.data_cspc = cspc | ||||
|
Alexander Valdez
|
r591 | self.dataOut.utctime = self.utcfirst | ||
| self.dataOut.flagNoData = False | ||||
|
Alexander Valdez
|
r593 | |||
| return 1 | ||||
|
Percy Condor
|
r1383 | |||