diff --git a/schainpy/model/io/jroIO_matlab.py b/schainpy/model/io/jroIO_matlab.py new file mode 100644 index 0000000..a0c14fd --- /dev/null +++ b/schainpy/model/io/jroIO_matlab.py @@ -0,0 +1,337 @@ +from schainpy.model.data.jrodata import * +from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation +from schainpy.model.io.jroIO_base import * + +import scipy.io as sio +import pprint +import numpy as np +from os import listdir +from os.path import isfile, join +import datetime +import cmath +from astropy.io.ascii.tests.test_connect import files + +# Path needs to be whereever the matlab data files are stored. +# This program works by reading how many data folders remain inside of the +# path destination. Every iteration of the program it will re-search +# the path destination, if new folder(s) are detected, it will read +# in all data from say... enough files to cover an hour and generate +# spectra and RTI plots. + +class MatReader(ProcessingUnit): + + index=None + list=None + firsttime=True + utccounter=None + utcfiletime=None + utcmatcounter=0 + utcfirst=None + utclist=None + foldercountercrosscheck=None + foldercountercheck=None + indexfirsttime=-31 + + def __init__(self): + self.dataOut = Spectra() + return + +# def FIRSTTIMERUNTHROUGH(self,path=None,startDate=None, endDate=None,startTime=datetime.time(0,0,0), +# endTime=datetime.time(23,59,59),walk=True,timezone='ut', +# all=0,online=True,ext=None,filelist=None, **kwargs): +# +# foldcountercrosscheck=0 +# self.foldercountercrosscheck=foldcountercrosscheck +# +# for g in range(len(filelist)): +# strsplit=filelist[g].split('.') +# timeints=[int(i) for i in strsplit] +# timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5]) +# utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds() +# secondlist.append(filelist[g]) +# self.utclist.append(utctime) +# +# for k in range(len(secondlist)): +# +# path1=os.path.join(self.path,secondlist[k]) +# filecounter=len([name for name in os.listdir(path1)]) +# # print "Reading from this dir:" +path1 +# for r in range(filecounter): +# matname=str(r)+'.mat' +# bork=os.path.join(path1,matname) +# # thirdlist is what ends up being the list to all matlab files +# thirdlist.append(os.path.join(path1,matname)) +# +# self.utcfirst=utclist[-1] +# self.index=-31 +# self.list=thirdlist +# +# while (self.index <= -1): +# currentfilenewest=self.list[self.index] +# print "Reading from this file:" + currentfilenewest +# filesplit=currentfilenewest.split("\\") +# newsplit=filesplit[-2] +# newnewsplit=newsplit.split(".") +# newnewsplit=[int(i) for i in newnewsplit] +# gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5]) +# self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds() +# +# +# newsplit=filesplit[-1] +# newnewsplit=newsplit.split(".") +# goobnum=newnewsplit[0] +# goobnum=int(goobnum) +# +# self.utcfirst=self.utcfirst+goobnum*2 +# print self.utcfirst +# +# datastuff=sio.loadmat(currentfilenewest) +# dataphase=datastuff.get('phase') +# data3=datastuff.get('doppler0') +# data4=datastuff.get('doppler1') +# data3= np.array(data3) +# data4 = np.array(data4) +# datacoh=datastuff.get('coherence2') +# +# datacohphase=datacoh*np.exp(-dataphase*1j) +# # data31 = np.fliplr(data3) +# # data41 = np.fliplr(data4) +# +# data31 = data3.reshape((1,data3.shape[0],data3.shape[1])) +# data41 = data4.reshape((1,data4.shape[0],data4.shape[1])) +# datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1])) +# +# datastack = np.vstack((data31,data41)) +# +# self.dataOut.pairsList=[(0,1)] +# self.dataOut.data_cspc=datacohphase1.copy() +# self.dataOut.data_spc = datastack.copy() +# self.dataOut.channelList = range(2) +# self.dataOut.nProfiles = 25 #this! +# self.dataOut.nIncohInt = 1 +# self.dataOut.nCohInt = 1 #this! +# self.dataOut.ippSeconds = 0.004 #this! +# self.dataOut.nFFTPoints = 25 +# self.dataOut.utctime = self.utcfirst +# self.dataOut.heightList = np.array(datastuff.get('hts')) +# +# self.dataOut.flagNoData = False +# +# self.firsttime=False +# +# self.index+=1 +# +# +# return foldercountercrosscheck, self.firsttime + + def run(self,path=None,startDate=None, endDate=None,startTime=datetime.time(0,0,0), + endTime=datetime.time(23,59,59),walk=True,timezone='ut', + all=0,online=True,ext=None, **kwargs): + + self.path=path + self.ext=ext + self.startDate=startDate + self.endDate=endDate + self.startTime=startTime + self.endTime=endTime + self.index=0 + + filelist = [ f for f in listdir(path)] + secondlist=[] + thirdlist=[] + utclist=[] + foldercountercheck=len([name for name in os.listdir(self.path)]) + +# ______________________________________________________________________________________________________________________________________________________________- +# First time through this acts much like the offline program, +# It reads in the latest file in the path to create a list. So, it finds +# the very last folder in the path, opens it and reads in the 30 matlab files +# inside of that. Then, this loop never runs again. + + + if (self.firsttime==True): +# self.FIRSTTIMERUNTHROUGH(path, self.startDate, self.endDate, self.startTime, +# self.endTime, walk, timezone, all, online, ext,filelist) + foldercountercrosscheck=0 + self.utclist=utclist + + + for g in range(len(filelist)): + strsplit=filelist[g].split('.') + timeints=[int(i) for i in strsplit] + timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5]) + utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds() + secondlist.append(filelist[g]) + self.utclist.append(utctime) + + for k in range(len(secondlist)): + + path1=os.path.join(self.path,secondlist[k]) + filecounter=len([name for name in os.listdir(path1)]) +# print "Reading from this dir:" +path1 + for r in range(filecounter): + matname=str(r)+'.mat' + bork=os.path.join(path1,matname) +# thirdlist is what ends up being the list to all matlab files + thirdlist.append(os.path.join(path1,matname)) + +# Set the index to -31 initially such that it reads the final 30 matlab files in the +# path for the first iteration. It gets reset in the continually checking portion. + + self.utcfirst=utclist[-1] + self.list=thirdlist + + currentfilenewest=self.list[self.indexfirsttime] + print "Reading from this file:" + currentfilenewest + filesplit=currentfilenewest.split("\\") + newsplit=filesplit[-2] + newnewsplit=newsplit.split(".") + newnewsplit=[int(i) for i in newnewsplit] + gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5]) + self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds() + + + newsplit=filesplit[-1] + newnewsplit=newsplit.split(".") + goobnum=newnewsplit[0] + goobnum=int(goobnum) + + self.utcfirst=self.utcfirst+goobnum*2 + print self.utcfirst + + datastuff=sio.loadmat(currentfilenewest) + dataphase=datastuff.get('phase') + data3=datastuff.get('doppler0') + data4=datastuff.get('doppler1') + data3= np.array(data3) + data4 = np.array(data4) + datacoh=datastuff.get('coherence2') + + datacohphase=datacoh*np.exp(-dataphase*1j) + # data31 = np.fliplr(data3) + # data41 = np.fliplr(data4) + + data31 = data3.reshape((1,data3.shape[0],data3.shape[1])) + data41 = data4.reshape((1,data4.shape[0],data4.shape[1])) + datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1])) + + datastack = np.vstack((data31,data41)) + + self.dataOut.pairsList=[(0,1)] + self.dataOut.data_cspc=datacohphase1.copy() + self.dataOut.data_spc = datastack.copy() + self.dataOut.channelList = range(2) + self.dataOut.nProfiles = 25 #this! + self.dataOut.nIncohInt = 1 + self.dataOut.nCohInt = 1 #this! + self.dataOut.ippSeconds = 0.004 #this! + self.dataOut.nFFTPoints = 25 + self.dataOut.utctime = self.utcfirst + self.dataOut.heightList = np.array(datastuff.get('hts')) + + self.dataOut.flagNoData = False + self.indexfirsttime+=1 + + if (self.indexfirsttime>=30): + self.firsttime=False + self.index=0 + + +# + + + +# __________________________________________________________________________________________________________________________________________________________- + + +# Now we check for new folders, if some are detected, we repeat the process + self.firsttime=False + + if(foldercountercheck>foldercountercrosscheck): + foldercountercrosscheck=foldercountercheck + + + for g in range(len(filelist)): + strsplit=filelist[g].split('.') + timeints=[int(i) for i in strsplit] + timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5]) + utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds() + secondlist.append(filelist[g]) + self.utclist.append(utctime) + + for k in range(len(secondlist)): + + path1=os.path.join(self.path,secondlist[k]) + filecounter=len([name for name in os.listdir(path1)]) +# print "Reading from this dir:" +path1 + for r in range(filecounter): + matname=str(r)+'.mat' + bork=os.path.join(path1,matname) +# thirdlist is what ends up being the list to all matlab files + thirdlist.append(os.path.join(path1,matname)) + +# Set the index to -31 initially such that it reads the final 30 matlab files in the +# path for the first iteration. It gets reset in the continually checking portion. + + self.utcfirst=utclist[-1] + self.list=thirdlist + + currentfilenewest=self.list[self.indexfirsttime] + print "Reading from this file:" + currentfilenewest + filesplit=currentfilenewest.split("\\") + newsplit=filesplit[-2] + newnewsplit=newsplit.split(".") + newnewsplit=[int(i) for i in newnewsplit] + gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5]) + self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds() + + + newsplit=filesplit[-1] + newnewsplit=newsplit.split(".") + goobnum=newnewsplit[0] + goobnum=int(goobnum) + + self.utcfirst=self.utcfirst+goobnum*2 + print "The utc of the file is:" + self.utcfirst + + datastuff=sio.loadmat(currentfilenewest) + dataphase=datastuff.get('phase') + data3=datastuff.get('doppler0') + data4=datastuff.get('doppler1') + data3= np.array(data3) + data4 = np.array(data4) + datacoh=datastuff.get('coherence2') + + datacohphase=datacoh*np.exp(-dataphase*1j) + # data31 = np.fliplr(data3) + # data41 = np.fliplr(data4) + + data31 = data3.reshape((1,data3.shape[0],data3.shape[1])) + data41 = data4.reshape((1,data4.shape[0],data4.shape[1])) + datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1])) + + datastack = np.vstack((data31,data41)) + + self.dataOut.pairsList=[(0,1)] + self.dataOut.data_cspc=datacohphase1.copy() + self.dataOut.data_spc = datastack.copy() + self.dataOut.channelList = range(2) + self.dataOut.nProfiles = 25 #this! + self.dataOut.nIncohInt = 1 + self.dataOut.nCohInt = 1 #this! + self.dataOut.ippSeconds = 0.004 #this! + self.dataOut.nFFTPoints = 25 + self.dataOut.utctime = self.utcfirst + self.dataOut.heightList = np.array(datastuff.get('hts')) + + self.dataOut.flagNoData = False + self.indexfirsttime+=1 + + if (self.indexfirsttime>=30): + self.firsttime=False + self.index=0 + + else: + time.sleep(15) + + return 1 \ No newline at end of file