diff --git a/schainpy/model/io/jroIO_matlab.py b/schainpy/model/io/jroIO_matlab.py index a0c14fd..b89ed97 100644 --- a/schainpy/model/io/jroIO_matlab.py +++ b/schainpy/model/io/jroIO_matlab.py @@ -9,16 +9,8 @@ from os import listdir from os.path import isfile, join import datetime import cmath -from astropy.io.ascii.tests.test_connect import files -# Path needs to be whereever the matlab data files are stored. -# This program works by reading how many data folders remain inside of the -# path destination. Every iteration of the program it will re-search -# the path destination, if new folder(s) are detected, it will read -# in all data from say... enough files to cover an hour and generate -# spectra and RTI plots. - -class MatReader(ProcessingUnit): +class matoffReader(ProcessingUnit): index=None list=None @@ -28,310 +20,246 @@ class MatReader(ProcessingUnit): utcmatcounter=0 utcfirst=None utclist=None - foldercountercrosscheck=None - foldercountercheck=None - indexfirsttime=-31 def __init__(self): self.dataOut = Spectra() + self.online = False return -# def FIRSTTIMERUNTHROUGH(self,path=None,startDate=None, endDate=None,startTime=datetime.time(0,0,0), -# endTime=datetime.time(23,59,59),walk=True,timezone='ut', -# all=0,online=True,ext=None,filelist=None, **kwargs): -# -# foldcountercrosscheck=0 -# self.foldercountercrosscheck=foldcountercrosscheck -# -# for g in range(len(filelist)): -# strsplit=filelist[g].split('.') -# timeints=[int(i) for i in strsplit] -# timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5]) -# utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds() -# secondlist.append(filelist[g]) -# self.utclist.append(utctime) -# -# for k in range(len(secondlist)): -# -# path1=os.path.join(self.path,secondlist[k]) -# filecounter=len([name for name in os.listdir(path1)]) -# # print "Reading from this dir:" +path1 -# for r in range(filecounter): -# matname=str(r)+'.mat' -# bork=os.path.join(path1,matname) -# # thirdlist is what ends up being the list to all matlab files -# thirdlist.append(os.path.join(path1,matname)) -# -# self.utcfirst=utclist[-1] -# self.index=-31 -# self.list=thirdlist -# -# while (self.index <= -1): -# currentfilenewest=self.list[self.index] -# print "Reading from this file:" + currentfilenewest -# filesplit=currentfilenewest.split("\\") -# newsplit=filesplit[-2] -# newnewsplit=newsplit.split(".") -# newnewsplit=[int(i) for i in newnewsplit] -# gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5]) -# self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds() -# -# -# newsplit=filesplit[-1] -# newnewsplit=newsplit.split(".") -# goobnum=newnewsplit[0] -# goobnum=int(goobnum) -# -# self.utcfirst=self.utcfirst+goobnum*2 -# print self.utcfirst -# -# datastuff=sio.loadmat(currentfilenewest) -# dataphase=datastuff.get('phase') -# data3=datastuff.get('doppler0') -# data4=datastuff.get('doppler1') -# data3= np.array(data3) -# data4 = np.array(data4) -# datacoh=datastuff.get('coherence2') -# -# datacohphase=datacoh*np.exp(-dataphase*1j) -# # data31 = np.fliplr(data3) -# # data41 = np.fliplr(data4) -# -# data31 = data3.reshape((1,data3.shape[0],data3.shape[1])) -# data41 = data4.reshape((1,data4.shape[0],data4.shape[1])) -# datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1])) -# -# datastack = np.vstack((data31,data41)) -# -# self.dataOut.pairsList=[(0,1)] -# self.dataOut.data_cspc=datacohphase1.copy() -# self.dataOut.data_spc = datastack.copy() -# self.dataOut.channelList = range(2) -# self.dataOut.nProfiles = 25 #this! -# self.dataOut.nIncohInt = 1 -# self.dataOut.nCohInt = 1 #this! -# self.dataOut.ippSeconds = 0.004 #this! -# self.dataOut.nFFTPoints = 25 -# self.dataOut.utctime = self.utcfirst -# self.dataOut.heightList = np.array(datastuff.get('hts')) -# -# self.dataOut.flagNoData = False -# -# self.firsttime=False -# -# self.index+=1 -# -# -# return foldercountercrosscheck, self.firsttime + def __setHeader(self, datastuff): + + self.dataOut.pairsList=[(0,1)] + self.dataOut.channelList = range(2) + self.dataOut.nProfiles = 25 #this! + self.dataOut.nIncohInt = 1 + self.dataOut.nCohInt = 1 #this! + self.dataOut.ippSeconds = 0.004 #this! + self.dataOut.nFFTPoints = 25 + self.dataOut.timeZone = 0 + self.dataOut.heightList = np.array(datastuff.get('hts')).flatten() - def run(self,path=None,startDate=None, endDate=None,startTime=datetime.time(0,0,0), - endTime=datetime.time(23,59,59),walk=True,timezone='ut', - all=0,online=True,ext=None, **kwargs): + def __readFile(self, currentfile): + print "Reading from this file:" + currentfile - self.path=path - self.ext=ext - self.startDate=startDate - self.endDate=endDate - self.startTime=startTime - self.endTime=endTime - self.index=0 + #filesplit=currentfile.split("\\") + filesplit=currentfile.split("/") + newsplit=filesplit[-2] + newnewsplit=newsplit.split(".") + newnewsplit=[int(i) for i in newnewsplit] + gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5]) + self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds() + + + newsplit=filesplit[-1] + newnewsplit=newsplit.split(".") + goobnum=newnewsplit[0] + goobnum=int(goobnum) + + self.utcfirst=self.utcfirst+goobnum*2 +# if (currentfile[43:]=='0.mat'): +# self.utcmatcounter=0 +# self.utcfirst=self.utclist[self.index] + +# if (self.utcmatcounter>60): +# self.utcmatcounter=0 + +# print self.utcmatcounter + print self.utcfirst + try: + datastuff=sio.loadmat(currentfile) + except: + return None, None + + dataphase=datastuff.get('phase') + data3=datastuff.get('doppler0') + data4=datastuff.get('doppler1') + data3= np.array(data3) + data4 = np.array(data4) + datacoh=datastuff.get('coherence2') + + datacohphase=datacoh*np.exp(-dataphase*1j) +# data31 = np.fliplr(data3) +# data41 = np.fliplr(data4) + + data31 = data3.reshape((1,data3.shape[0],data3.shape[1])) + data41 = data4.reshape((1,data4.shape[0],data4.shape[1])) + datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1])) + + datastack = np.vstack((data31,data41)) + + self.__setHeader(datastuff) + + spc = datastack + cspc = datacohphase1 + + return spc, cspc + + def __findFiles(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)): - filelist = [ f for f in listdir(path)] - secondlist=[] - thirdlist=[] + if startDate == None: + startDate = datetime.date(1970,1,1) + + if endDate == None: + endDate = datetime.date(2050,1,1) + + startsearch1=datetime.datetime.combine(startDate,startTime) + startsearch2=(startsearch1-datetime.datetime(1970,1,1)).total_seconds() + endsearch1=datetime.datetime.combine(endDate,endTime) + endsearch2=(endsearch1-datetime.datetime(1970,1,1)).total_seconds() + + dirList = listdir(path) + dirList = sorted(dirList) + + dirListFiltered=[] + fileListFiltered=[] utclist=[] - foldercountercheck=len([name for name in os.listdir(self.path)]) -# ______________________________________________________________________________________________________________________________________________________________- -# First time through this acts much like the offline program, -# It reads in the latest file in the path to create a list. So, it finds -# the very last folder in the path, opens it and reads in the 30 matlab files -# inside of that. Then, this loop never runs again. - - - if (self.firsttime==True): -# self.FIRSTTIMERUNTHROUGH(path, self.startDate, self.endDate, self.startTime, -# self.endTime, walk, timezone, all, online, ext,filelist) - foldercountercrosscheck=0 - self.utclist=utclist + if not dirList: + print "No directories found" + return [] + + for thisDir in dirList: + if not os.path.isdir(os.path.join(path, thisDir)): + continue + + strsplit=thisDir.split('.') + timeints=[int(i) for i in strsplit] + timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5]) + utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds() + if not self.online: + if (utctime > endsearch2): + continue + + if (utctime < startsearch2): + continue - for g in range(len(filelist)): - strsplit=filelist[g].split('.') - timeints=[int(i) for i in strsplit] - timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5]) - utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds() - secondlist.append(filelist[g]) - self.utclist.append(utctime) - - for k in range(len(secondlist)): - - path1=os.path.join(self.path,secondlist[k]) - filecounter=len([name for name in os.listdir(path1)]) -# print "Reading from this dir:" +path1 - for r in range(filecounter): - matname=str(r)+'.mat' - bork=os.path.join(path1,matname) -# thirdlist is what ends up being the list to all matlab files - thirdlist.append(os.path.join(path1,matname)) + dirListFiltered.append(thisDir) + utclist.append(utctime) + + if not dirListFiltered: + print "filtro" + return [] + + for thisDir in dirListFiltered: + + pathFile = os.path.join(self.path, thisDir) + + fileList = os.listdir(pathFile) + + if not fileList: + continue -# Set the index to -31 initially such that it reads the final 30 matlab files in the -# path for the first iteration. It gets reset in the continually checking portion. + for k in range(len(fileList)): + thisFile = str(k)+'.mat' + + if not os.path.isfile(os.path.join(pathFile, thisFile)): + continue + + fileListFiltered.append(os.path.join(pathFile, thisFile)) + + return fileListFiltered + + def __getNextOnlineFile(self, seconds = 30): + + filename = self.__getNextOfflineFile() + + if filename: + return filename + + ncurrentfiles = len(self.fileList) + + nTries = 0 + while (True): + filelist = self.__findFiles(self.path) + if len(filelist) > ncurrentfiles: + break - self.utcfirst=utclist[-1] - self.list=thirdlist - - currentfilenewest=self.list[self.indexfirsttime] - print "Reading from this file:" + currentfilenewest - filesplit=currentfilenewest.split("\\") - newsplit=filesplit[-2] - newnewsplit=newsplit.split(".") - newnewsplit=[int(i) for i in newnewsplit] - gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5]) - self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds() - - - newsplit=filesplit[-1] - newnewsplit=newsplit.split(".") - goobnum=newnewsplit[0] - goobnum=int(goobnum) - - self.utcfirst=self.utcfirst+goobnum*2 - print self.utcfirst - - datastuff=sio.loadmat(currentfilenewest) - dataphase=datastuff.get('phase') - data3=datastuff.get('doppler0') - data4=datastuff.get('doppler1') - data3= np.array(data3) - data4 = np.array(data4) - datacoh=datastuff.get('coherence2') - - datacohphase=datacoh*np.exp(-dataphase*1j) - # data31 = np.fliplr(data3) - # data41 = np.fliplr(data4) - - data31 = data3.reshape((1,data3.shape[0],data3.shape[1])) - data41 = data4.reshape((1,data4.shape[0],data4.shape[1])) - datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1])) - - datastack = np.vstack((data31,data41)) - - self.dataOut.pairsList=[(0,1)] - self.dataOut.data_cspc=datacohphase1.copy() - self.dataOut.data_spc = datastack.copy() - self.dataOut.channelList = range(2) - self.dataOut.nProfiles = 25 #this! - self.dataOut.nIncohInt = 1 - self.dataOut.nCohInt = 1 #this! - self.dataOut.ippSeconds = 0.004 #this! - self.dataOut.nFFTPoints = 25 - self.dataOut.utctime = self.utcfirst - self.dataOut.heightList = np.array(datastuff.get('hts')) - - self.dataOut.flagNoData = False - self.indexfirsttime+=1 + nTries += 1 - if (self.indexfirsttime>=30): - self.firsttime=False - self.index=0 - + if nTries > 3: + break -# + print "Waiting %d seconds ..." %seconds + time.sleep(30) + + if not (len(filelist) > ncurrentfiles): + return None + self.fileList = filelist + filename = self.__getNextOfflineFile() + return filename + + def __getNextOfflineFile(self): + + if self.index >= len(self.fileList): + return None + + filename=self.fileList[self.index] + self.index += 1 + return filename + + def __getNextFile(self): -# __________________________________________________________________________________________________________________________________________________________- + if self.online: + filename = self.__getNextOnlineFile() + else: + filename = self.__getNextOfflineFile() + return filename + def setup(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)): + + fileList = self.__findFiles(path, startDate, endDate, startTime, endTime) + + if self.online: + self.index = len(fileList) -1 + else: + self.index = 0 -# Now we check for new folders, if some are detected, we repeat the process - self.firsttime=False - - if(foldercountercheck>foldercountercrosscheck): - foldercountercrosscheck=foldercountercheck + self.fileList = fileList + + print "fin setup" + + def run(self,path=None,startDate=None, endDate=None, + startTime=datetime.time(0,0,0), + endTime=datetime.time(23,59,59), + walk=True,timezone='ut', + all=0,online=False,ext=None,**kwargs): + + self.path=path + self.ext=ext + self.startDate=startDate + self.endDate=endDate + self.startTime=startTime + self.endTime=endTime + self.online = online + self.dataOut.flagNoData = True + + if (self.firsttime==True): + self.setup(path, startDate, endDate, startTime, endTime) + self.firsttime=False - - for g in range(len(filelist)): - strsplit=filelist[g].split('.') - timeints=[int(i) for i in strsplit] - timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5]) - utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds() - secondlist.append(filelist[g]) - self.utclist.append(utctime) - - for k in range(len(secondlist)): - - path1=os.path.join(self.path,secondlist[k]) - filecounter=len([name for name in os.listdir(path1)]) -# print "Reading from this dir:" +path1 - for r in range(filecounter): - matname=str(r)+'.mat' - bork=os.path.join(path1,matname) -# thirdlist is what ends up being the list to all matlab files - thirdlist.append(os.path.join(path1,matname)) + + if not self.fileList: + self.dataOut.flagNoData = True + print "lista vacia" + return + + currentfile = self.__getNextFile() + + if not currentfile: + "no file" + return + + spc, cspc = self.__readFile(currentfile) + + if spc!=None: -# Set the index to -31 initially such that it reads the final 30 matlab files in the -# path for the first iteration. It gets reset in the continually checking portion. - - self.utcfirst=utclist[-1] - self.list=thirdlist - - currentfilenewest=self.list[self.indexfirsttime] - print "Reading from this file:" + currentfilenewest - filesplit=currentfilenewest.split("\\") - newsplit=filesplit[-2] - newnewsplit=newsplit.split(".") - newnewsplit=[int(i) for i in newnewsplit] - gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5]) - self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds() - - - newsplit=filesplit[-1] - newnewsplit=newsplit.split(".") - goobnum=newnewsplit[0] - goobnum=int(goobnum) - - self.utcfirst=self.utcfirst+goobnum*2 - print "The utc of the file is:" + self.utcfirst - - datastuff=sio.loadmat(currentfilenewest) - dataphase=datastuff.get('phase') - data3=datastuff.get('doppler0') - data4=datastuff.get('doppler1') - data3= np.array(data3) - data4 = np.array(data4) - datacoh=datastuff.get('coherence2') - - datacohphase=datacoh*np.exp(-dataphase*1j) - # data31 = np.fliplr(data3) - # data41 = np.fliplr(data4) - - data31 = data3.reshape((1,data3.shape[0],data3.shape[1])) - data41 = data4.reshape((1,data4.shape[0],data4.shape[1])) - datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1])) - - datastack = np.vstack((data31,data41)) - - self.dataOut.pairsList=[(0,1)] - self.dataOut.data_cspc=datacohphase1.copy() - self.dataOut.data_spc = datastack.copy() - self.dataOut.channelList = range(2) - self.dataOut.nProfiles = 25 #this! - self.dataOut.nIncohInt = 1 - self.dataOut.nCohInt = 1 #this! - self.dataOut.ippSeconds = 0.004 #this! - self.dataOut.nFFTPoints = 25 + self.dataOut.data_spc = spc + self.dataOut.data_cspc = cspc self.dataOut.utctime = self.utcfirst - self.dataOut.heightList = np.array(datastuff.get('hts')) - self.dataOut.flagNoData = False - self.indexfirsttime+=1 - - if (self.indexfirsttime>=30): - self.firsttime=False - self.index=0 - - else: - time.sleep(15) - - return 1 \ No newline at end of file + + return 1 + \ No newline at end of file