|
|
from schainpy.model.data.jrodata import *
|
|
|
from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
|
|
|
from schainpy.model.io.jroIO_base import *
|
|
|
|
|
|
import scipy.io as sio
|
|
|
import pprint
|
|
|
import numpy as np
|
|
|
from os import listdir
|
|
|
from os.path import isfile, join
|
|
|
import datetime
|
|
|
import cmath
|
|
|
|
|
|
class matoffReader(ProcessingUnit):
|
|
|
|
|
|
index=None
|
|
|
list=None
|
|
|
firsttime=True
|
|
|
utccounter=None
|
|
|
utcfiletime=None
|
|
|
utcmatcounter=0
|
|
|
utcfirst=None
|
|
|
utclist=None
|
|
|
|
|
|
def __init__(self):
|
|
|
self.dataOut = Spectra()
|
|
|
self.online = False
|
|
|
return
|
|
|
|
|
|
def __setHeader(self, datastuff):
|
|
|
|
|
|
self.dataOut.pairsList=[(0,1)]
|
|
|
self.dataOut.channelList = list(range(np.array(datastuff.get('power')).shape[1]))
|
|
|
self.dataOut.nProfiles = len(np.array(datastuff.get('vel')).flatten()) #this!
|
|
|
self.dataOut.nIncohInt = 20
|
|
|
self.dataOut.nCohInt = 1 #this!
|
|
|
self.dataOut.ippSeconds = 0.004 #this!
|
|
|
self.dataOut.nFFTPoints = len(np.array(datastuff.get('vel')).flatten())
|
|
|
self.dataOut.timeZone = 0
|
|
|
self.dataOut.heightList = np.array(datastuff.get('hts')).flatten()
|
|
|
|
|
|
def __readFile(self, currentfile):
|
|
|
print("Reading from this file:" + currentfile)
|
|
|
|
|
|
#filesplit=currentfile.split("\\")
|
|
|
filesplit=currentfile.split("/")
|
|
|
newsplit=filesplit[-2]
|
|
|
newnewsplit=newsplit.split(".")
|
|
|
newnewsplit=[int(i) for i in newnewsplit]
|
|
|
gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5])
|
|
|
self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds()
|
|
|
|
|
|
|
|
|
newsplit=filesplit[-1]
|
|
|
newnewsplit=newsplit.split(".")
|
|
|
goobnum=newnewsplit[0]
|
|
|
goobnum=int(goobnum)
|
|
|
|
|
|
self.utcfirst=self.utcfirst+goobnum*2
|
|
|
# if (currentfile[43:]=='0.mat'):
|
|
|
# self.utcmatcounter=0
|
|
|
# self.utcfirst=self.utclist[self.index]
|
|
|
|
|
|
# if (self.utcmatcounter>60):
|
|
|
# self.utcmatcounter=0
|
|
|
|
|
|
# print self.utcmatcounter
|
|
|
print(self.utcfirst)
|
|
|
try:
|
|
|
datastuff=sio.loadmat(currentfile)
|
|
|
except:
|
|
|
return None, None
|
|
|
|
|
|
dataphase=datastuff.get('phase')
|
|
|
data3=datastuff.get('doppler0')
|
|
|
data4=datastuff.get('doppler1')
|
|
|
data3= np.array(data3)
|
|
|
data4 = np.array(data4)
|
|
|
datacoh=datastuff.get('coherence2')
|
|
|
|
|
|
datacohphase=datacoh*np.exp(-dataphase*1j)
|
|
|
# data31 = np.fliplr(data3)
|
|
|
# data41 = np.fliplr(data4)
|
|
|
|
|
|
data31 = data3.reshape((1,data3.shape[0],data3.shape[1]))
|
|
|
data41 = data4.reshape((1,data4.shape[0],data4.shape[1]))
|
|
|
datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1]))
|
|
|
|
|
|
datastack = np.vstack((data31,data41))
|
|
|
|
|
|
self.__setHeader(datastuff)
|
|
|
|
|
|
spc = datastack
|
|
|
cspc = datacohphase1
|
|
|
|
|
|
return spc, cspc
|
|
|
|
|
|
def __findFiles(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
|
|
|
|
|
|
if startDate == None:
|
|
|
startDate = datetime.date(1970,1,1)
|
|
|
|
|
|
if endDate == None:
|
|
|
endDate = datetime.date(2050,1,1)
|
|
|
|
|
|
startsearch1=datetime.datetime.combine(startDate,startTime)
|
|
|
startsearch2=(startsearch1-datetime.datetime(1970,1,1)).total_seconds()
|
|
|
endsearch1=datetime.datetime.combine(endDate,endTime)
|
|
|
endsearch2=(endsearch1-datetime.datetime(1970,1,1)).total_seconds()
|
|
|
|
|
|
dirList = listdir(path)
|
|
|
dirList = sorted(dirList)
|
|
|
|
|
|
dirListFiltered=[]
|
|
|
fileListFiltered=[]
|
|
|
utclist=[]
|
|
|
|
|
|
if not dirList:
|
|
|
print("No directories found")
|
|
|
return []
|
|
|
|
|
|
#if self.online:
|
|
|
# dirList= [dirList[-1]]
|
|
|
|
|
|
if self.online:
|
|
|
currentdate = datetime.datetime.now()
|
|
|
strsplit1=currentdate.strftime('%Y.%m.%d')
|
|
|
dirList = fnmatch.filter(dirList,strsplit1+'*')
|
|
|
|
|
|
for thisDir in dirList:
|
|
|
if not os.path.isdir(os.path.join(path, thisDir)):
|
|
|
continue
|
|
|
|
|
|
strsplit=thisDir.split('.')
|
|
|
timeints=[int(i) for i in strsplit]
|
|
|
timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
|
|
|
utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
|
|
|
|
|
|
if not self.online:
|
|
|
if (utctime > endsearch2):
|
|
|
continue
|
|
|
|
|
|
if (utctime < startsearch2):
|
|
|
continue
|
|
|
|
|
|
dirListFiltered.append(thisDir)
|
|
|
utclist.append(utctime)
|
|
|
|
|
|
if not dirListFiltered:
|
|
|
print("filtro")
|
|
|
return []
|
|
|
|
|
|
for thisDir in dirListFiltered:
|
|
|
|
|
|
pathFile = os.path.join(self.path, thisDir)
|
|
|
|
|
|
fileList = os.listdir(pathFile)
|
|
|
|
|
|
if not fileList:
|
|
|
continue
|
|
|
|
|
|
for k in range(len(fileList)):
|
|
|
thisFile = str(k)+'.mat'
|
|
|
|
|
|
if not os.path.isfile(os.path.join(pathFile, thisFile)):
|
|
|
continue
|
|
|
|
|
|
fileListFiltered.append(os.path.join(pathFile, thisFile))
|
|
|
|
|
|
return fileListFiltered
|
|
|
|
|
|
def __getNextOnlineFile(self, seconds = 40):
|
|
|
|
|
|
filename = self.__getNextOfflineFile()
|
|
|
|
|
|
if filename:
|
|
|
return filename
|
|
|
|
|
|
ncurrentfiles = len(self.fileList)
|
|
|
|
|
|
nTries = 0
|
|
|
while (True):
|
|
|
filelist = self.__findFiles(self.path)
|
|
|
if len(filelist) > ncurrentfiles:
|
|
|
break
|
|
|
|
|
|
nTries += 1
|
|
|
|
|
|
if nTries > 3:
|
|
|
break
|
|
|
|
|
|
print("Waiting %d seconds ..." %seconds)
|
|
|
time.sleep(40)
|
|
|
|
|
|
if not (len(filelist) > ncurrentfiles):
|
|
|
return None
|
|
|
|
|
|
self.fileList = filelist
|
|
|
filename = self.__getNextOfflineFile()
|
|
|
|
|
|
return filename
|
|
|
|
|
|
def __getNextOfflineFile(self):
|
|
|
|
|
|
if self.index >= len(self.fileList):
|
|
|
return None
|
|
|
|
|
|
filename=self.fileList[self.index]
|
|
|
self.index += 1
|
|
|
return filename
|
|
|
|
|
|
def __getNextFile(self):
|
|
|
|
|
|
if self.online:
|
|
|
filename = self.__getNextOnlineFile()
|
|
|
else:
|
|
|
filename = self.__getNextOfflineFile()
|
|
|
return filename
|
|
|
|
|
|
def setup(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
|
|
|
|
|
|
fileList = self.__findFiles(path, startDate, endDate, startTime, endTime)
|
|
|
|
|
|
if self.online:
|
|
|
self.index = len(fileList) -1
|
|
|
else:
|
|
|
self.index = 0
|
|
|
|
|
|
self.fileList = fileList
|
|
|
|
|
|
print("fin setup")
|
|
|
|
|
|
def run(self,path=None,startDate=None, endDate=None,
|
|
|
startTime=datetime.time(0,0,0),
|
|
|
endTime=datetime.time(23,59,59),
|
|
|
walk=True,timezone='ut',
|
|
|
all=0,online=False,ext=None,**kwargs):
|
|
|
|
|
|
self.path=path
|
|
|
self.ext=ext
|
|
|
self.startDate=startDate
|
|
|
self.endDate=endDate
|
|
|
self.startTime=startTime
|
|
|
self.endTime=endTime
|
|
|
self.online = online
|
|
|
self.dataOut.flagNoData = True
|
|
|
|
|
|
if (self.firsttime==True):
|
|
|
self.setup(path, startDate, endDate, startTime, endTime)
|
|
|
self.firsttime=False
|
|
|
|
|
|
|
|
|
if not self.fileList:
|
|
|
self.dataOut.flagNoData = True
|
|
|
print("lista vacia")
|
|
|
return
|
|
|
|
|
|
currentfile = self.__getNextFile()
|
|
|
|
|
|
if not currentfile:
|
|
|
"no file"
|
|
|
return
|
|
|
|
|
|
spc, cspc = self.__readFile(currentfile)
|
|
|
|
|
|
if spc!=None:
|
|
|
|
|
|
self.dataOut.data_spc = spc
|
|
|
self.dataOut.data_cspc = cspc
|
|
|
self.dataOut.utctime = self.utcfirst
|
|
|
self.dataOut.flagNoData = False
|
|
|
|
|
|
return 1
|
|
|
|
