from schainpy.model.data.jrodata import * from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation from schainpy.model.io.jroIO_base import * import scipy.io as sio import pprint import numpy as np from os import listdir from os.path import isfile, join import datetime import cmath from astropy.io.ascii.tests.test_connect import files # Path needs to be whereever the matlab data files are stored. # This program works by reading how many data folders remain inside of the # path destination. Every iteration of the program it will re-search # the path destination, if new folder(s) are detected, it will read # in all data from say... enough files to cover an hour and generate # spectra and RTI plots. class MatReader(ProcessingUnit): index=None list=None firsttime=True utccounter=None utcfiletime=None utcmatcounter=0 utcfirst=None utclist=None foldercountercrosscheck=None foldercountercheck=None indexfirsttime=-31 def __init__(self): self.dataOut = Spectra() return # def FIRSTTIMERUNTHROUGH(self,path=None,startDate=None, endDate=None,startTime=datetime.time(0,0,0), # endTime=datetime.time(23,59,59),walk=True,timezone='ut', # all=0,online=True,ext=None,filelist=None, **kwargs): # # foldcountercrosscheck=0 # self.foldercountercrosscheck=foldcountercrosscheck # # for g in range(len(filelist)): # strsplit=filelist[g].split('.') # timeints=[int(i) for i in strsplit] # timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5]) # utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds() # secondlist.append(filelist[g]) # self.utclist.append(utctime) # # for k in range(len(secondlist)): # # path1=os.path.join(self.path,secondlist[k]) # filecounter=len([name for name in os.listdir(path1)]) # # print "Reading from this dir:" +path1 # for r in range(filecounter): # matname=str(r)+'.mat' # bork=os.path.join(path1,matname) # # thirdlist is what ends up being the list to all matlab files # thirdlist.append(os.path.join(path1,matname)) # # self.utcfirst=utclist[-1] # self.index=-31 # self.list=thirdlist # # while (self.index <= -1): # currentfilenewest=self.list[self.index] # print "Reading from this file:" + currentfilenewest # filesplit=currentfilenewest.split("\\") # newsplit=filesplit[-2] # newnewsplit=newsplit.split(".") # newnewsplit=[int(i) for i in newnewsplit] # gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5]) # self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds() # # # newsplit=filesplit[-1] # newnewsplit=newsplit.split(".") # goobnum=newnewsplit[0] # goobnum=int(goobnum) # # self.utcfirst=self.utcfirst+goobnum*2 # print self.utcfirst # # datastuff=sio.loadmat(currentfilenewest) # dataphase=datastuff.get('phase') # data3=datastuff.get('doppler0') # data4=datastuff.get('doppler1') # data3= np.array(data3) # data4 = np.array(data4) # datacoh=datastuff.get('coherence2') # # datacohphase=datacoh*np.exp(-dataphase*1j) # # data31 = np.fliplr(data3) # # data41 = np.fliplr(data4) # # data31 = data3.reshape((1,data3.shape[0],data3.shape[1])) # data41 = data4.reshape((1,data4.shape[0],data4.shape[1])) # datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1])) # # datastack = np.vstack((data31,data41)) # # self.dataOut.pairsList=[(0,1)] # self.dataOut.data_cspc=datacohphase1.copy() # self.dataOut.data_spc = datastack.copy() # self.dataOut.channelList = range(2) # self.dataOut.nProfiles = 25 #this! # self.dataOut.nIncohInt = 1 # self.dataOut.nCohInt = 1 #this! # self.dataOut.ippSeconds = 0.004 #this! # self.dataOut.nFFTPoints = 25 # self.dataOut.utctime = self.utcfirst # self.dataOut.heightList = np.array(datastuff.get('hts')) # # self.dataOut.flagNoData = False # # self.firsttime=False # # self.index+=1 # # # return foldercountercrosscheck, self.firsttime def run(self,path=None,startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59),walk=True,timezone='ut', all=0,online=True,ext=None, **kwargs): self.path=path self.ext=ext self.startDate=startDate self.endDate=endDate self.startTime=startTime self.endTime=endTime self.index=0 filelist = [ f for f in listdir(path)] secondlist=[] thirdlist=[] utclist=[] foldercountercheck=len([name for name in os.listdir(self.path)]) # ______________________________________________________________________________________________________________________________________________________________- # First time through this acts much like the offline program, # It reads in the latest file in the path to create a list. So, it finds # the very last folder in the path, opens it and reads in the 30 matlab files # inside of that. Then, this loop never runs again. if (self.firsttime==True): # self.FIRSTTIMERUNTHROUGH(path, self.startDate, self.endDate, self.startTime, # self.endTime, walk, timezone, all, online, ext,filelist) foldercountercrosscheck=0 self.utclist=utclist for g in range(len(filelist)): strsplit=filelist[g].split('.') timeints=[int(i) for i in strsplit] timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5]) utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds() secondlist.append(filelist[g]) self.utclist.append(utctime) for k in range(len(secondlist)): path1=os.path.join(self.path,secondlist[k]) filecounter=len([name for name in os.listdir(path1)]) # print "Reading from this dir:" +path1 for r in range(filecounter): matname=str(r)+'.mat' bork=os.path.join(path1,matname) # thirdlist is what ends up being the list to all matlab files thirdlist.append(os.path.join(path1,matname)) # Set the index to -31 initially such that it reads the final 30 matlab files in the # path for the first iteration. It gets reset in the continually checking portion. self.utcfirst=utclist[-1] self.list=thirdlist currentfilenewest=self.list[self.indexfirsttime] print "Reading from this file:" + currentfilenewest filesplit=currentfilenewest.split("\\") newsplit=filesplit[-2] newnewsplit=newsplit.split(".") newnewsplit=[int(i) for i in newnewsplit] gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5]) self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds() newsplit=filesplit[-1] newnewsplit=newsplit.split(".") goobnum=newnewsplit[0] goobnum=int(goobnum) self.utcfirst=self.utcfirst+goobnum*2 print self.utcfirst datastuff=sio.loadmat(currentfilenewest) dataphase=datastuff.get('phase') data3=datastuff.get('doppler0') data4=datastuff.get('doppler1') data3= np.array(data3) data4 = np.array(data4) datacoh=datastuff.get('coherence2') datacohphase=datacoh*np.exp(-dataphase*1j) # data31 = np.fliplr(data3) # data41 = np.fliplr(data4) data31 = data3.reshape((1,data3.shape[0],data3.shape[1])) data41 = data4.reshape((1,data4.shape[0],data4.shape[1])) datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1])) datastack = np.vstack((data31,data41)) self.dataOut.pairsList=[(0,1)] self.dataOut.data_cspc=datacohphase1.copy() self.dataOut.data_spc = datastack.copy() self.dataOut.channelList = range(2) self.dataOut.nProfiles = 25 #this! self.dataOut.nIncohInt = 1 self.dataOut.nCohInt = 1 #this! self.dataOut.ippSeconds = 0.004 #this! self.dataOut.nFFTPoints = 25 self.dataOut.utctime = self.utcfirst self.dataOut.heightList = np.array(datastuff.get('hts')) self.dataOut.flagNoData = False self.indexfirsttime+=1 if (self.indexfirsttime>=30): self.firsttime=False self.index=0 # # __________________________________________________________________________________________________________________________________________________________- # Now we check for new folders, if some are detected, we repeat the process self.firsttime=False if(foldercountercheck>foldercountercrosscheck): foldercountercrosscheck=foldercountercheck for g in range(len(filelist)): strsplit=filelist[g].split('.') timeints=[int(i) for i in strsplit] timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5]) utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds() secondlist.append(filelist[g]) self.utclist.append(utctime) for k in range(len(secondlist)): path1=os.path.join(self.path,secondlist[k]) filecounter=len([name for name in os.listdir(path1)]) # print "Reading from this dir:" +path1 for r in range(filecounter): matname=str(r)+'.mat' bork=os.path.join(path1,matname) # thirdlist is what ends up being the list to all matlab files thirdlist.append(os.path.join(path1,matname)) # Set the index to -31 initially such that it reads the final 30 matlab files in the # path for the first iteration. It gets reset in the continually checking portion. self.utcfirst=utclist[-1] self.list=thirdlist currentfilenewest=self.list[self.indexfirsttime] print "Reading from this file:" + currentfilenewest filesplit=currentfilenewest.split("\\") newsplit=filesplit[-2] newnewsplit=newsplit.split(".") newnewsplit=[int(i) for i in newnewsplit] gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5]) self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds() newsplit=filesplit[-1] newnewsplit=newsplit.split(".") goobnum=newnewsplit[0] goobnum=int(goobnum) self.utcfirst=self.utcfirst+goobnum*2 print "The utc of the file is:" + self.utcfirst datastuff=sio.loadmat(currentfilenewest) dataphase=datastuff.get('phase') data3=datastuff.get('doppler0') data4=datastuff.get('doppler1') data3= np.array(data3) data4 = np.array(data4) datacoh=datastuff.get('coherence2') datacohphase=datacoh*np.exp(-dataphase*1j) # data31 = np.fliplr(data3) # data41 = np.fliplr(data4) data31 = data3.reshape((1,data3.shape[0],data3.shape[1])) data41 = data4.reshape((1,data4.shape[0],data4.shape[1])) datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1])) datastack = np.vstack((data31,data41)) self.dataOut.pairsList=[(0,1)] self.dataOut.data_cspc=datacohphase1.copy() self.dataOut.data_spc = datastack.copy() self.dataOut.channelList = range(2) self.dataOut.nProfiles = 25 #this! self.dataOut.nIncohInt = 1 self.dataOut.nCohInt = 1 #this! self.dataOut.ippSeconds = 0.004 #this! self.dataOut.nFFTPoints = 25 self.dataOut.utctime = self.utcfirst self.dataOut.heightList = np.array(datastuff.get('hts')) self.dataOut.flagNoData = False self.indexfirsttime+=1 if (self.indexfirsttime>=30): self.firsttime=False self.index=0 else: time.sleep(15) return 1