|
|
from schainpy.model.data.jrodata import *
|
|
|
from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
|
|
|
from schainpy.model.io.jroIO_base import *
|
|
|
|
|
|
import scipy.io as sio
|
|
|
import pprint
|
|
|
import numpy as np
|
|
|
from os import listdir
|
|
|
from os.path import isfile, join
|
|
|
import datetime
|
|
|
import cmath
|
|
|
|
|
|
class matoffReader(ProcessingUnit):
|
|
|
|
|
|
index = None
|
|
|
list = None
|
|
|
firsttime = True
|
|
|
utccounter = None
|
|
|
utcfiletime = None
|
|
|
utcmatcounter = 0
|
|
|
utcfirst = None
|
|
|
utclist = None
|
|
|
|
|
|
def __init__(self):
|
|
|
self.dataOut = Spectra()
|
|
|
self.online = False
|
|
|
return
|
|
|
|
|
|
def __setHeader(self, datastuff):
|
|
|
|
|
|
self.dataOut.pairsList = [(0, 1)]
|
|
|
self.dataOut.channelList = list(range(np.array(datastuff.get('power')).shape[1]))
|
|
|
self.dataOut.nProfiles = len(np.array(datastuff.get('vel')).flatten()) # this!
|
|
|
self.dataOut.nIncohInt = 20
|
|
|
self.dataOut.nCohInt = 1 # this!
|
|
|
self.dataOut.ippSeconds = 0.004 # this!
|
|
|
self.dataOut.nFFTPoints = len(np.array(datastuff.get('vel')).flatten())
|
|
|
self.dataOut.timeZone = 0
|
|
|
self.dataOut.heightList = np.array(datastuff.get('hts')).flatten()
|
|
|
|
|
|
def __readFile(self, currentfile):
|
|
|
print("Reading from this file:" + currentfile)
|
|
|
|
|
|
# filesplit=currentfile.split("\\")
|
|
|
filesplit = currentfile.split("/")
|
|
|
newsplit = filesplit[-2]
|
|
|
newnewsplit = newsplit.split(".")
|
|
|
newnewsplit = [int(i) for i in newnewsplit]
|
|
|
gooblist = datetime.datetime(newnewsplit[0], newnewsplit[1], newnewsplit[2], newnewsplit[3], newnewsplit[4], newnewsplit[5])
|
|
|
self.utcfirst = (gooblist - datetime.datetime(1970, 1, 1)).total_seconds()
|
|
|
|
|
|
|
|
|
newsplit = filesplit[-1]
|
|
|
newnewsplit = newsplit.split(".")
|
|
|
goobnum = newnewsplit[0]
|
|
|
goobnum = int(goobnum)
|
|
|
|
|
|
self.utcfirst = self.utcfirst + goobnum * 2
|
|
|
# if (currentfile[43:]=='0.mat'):
|
|
|
# self.utcmatcounter=0
|
|
|
# self.utcfirst=self.utclist[self.index]
|
|
|
|
|
|
# if (self.utcmatcounter>60):
|
|
|
# self.utcmatcounter=0
|
|
|
|
|
|
# print self.utcmatcounter
|
|
|
print(self.utcfirst)
|
|
|
try:
|
|
|
datastuff = sio.loadmat(currentfile)
|
|
|
except:
|
|
|
return None, None
|
|
|
|
|
|
dataphase = datastuff.get('phase')
|
|
|
data3 = datastuff.get('doppler0')
|
|
|
data4 = datastuff.get('doppler1')
|
|
|
data3 = np.array(data3)
|
|
|
data4 = np.array(data4)
|
|
|
datacoh = datastuff.get('coherence2')
|
|
|
|
|
|
datacohphase = datacoh * np.exp(-dataphase * 1j)
|
|
|
# data31 = np.fliplr(data3)
|
|
|
# data41 = np.fliplr(data4)
|
|
|
|
|
|
data31 = data3.reshape((1, data3.shape[0], data3.shape[1]))
|
|
|
data41 = data4.reshape((1, data4.shape[0], data4.shape[1]))
|
|
|
datacohphase1 = datacohphase.reshape((1, datacoh.shape[0], datacoh.shape[1]))
|
|
|
|
|
|
datastack = np.vstack((data31, data41))
|
|
|
|
|
|
self.__setHeader(datastuff)
|
|
|
|
|
|
spc = datastack
|
|
|
cspc = datacohphase1
|
|
|
|
|
|
return spc, cspc
|
|
|
|
|
|
def __findFiles(self, path, startDate=None, endDate=None, startTime=datetime.time(0, 0, 0), endTime=datetime.time(23, 59, 59)):
|
|
|
|
|
|
if startDate == None:
|
|
|
startDate = datetime.date(1970, 1, 1)
|
|
|
|
|
|
if endDate == None:
|
|
|
endDate = datetime.date(2050, 1, 1)
|
|
|
|
|
|
startsearch1 = datetime.datetime.combine(startDate, startTime)
|
|
|
startsearch2 = (startsearch1 - datetime.datetime(1970, 1, 1)).total_seconds()
|
|
|
endsearch1 = datetime.datetime.combine(endDate, endTime)
|
|
|
endsearch2 = (endsearch1 - datetime.datetime(1970, 1, 1)).total_seconds()
|
|
|
|
|
|
dirList = listdir(path)
|
|
|
dirList = sorted(dirList)
|
|
|
|
|
|
dirListFiltered = []
|
|
|
fileListFiltered = []
|
|
|
utclist = []
|
|
|
|
|
|
if not dirList:
|
|
|
print("No directories found")
|
|
|
return []
|
|
|
|
|
|
# if self.online:
|
|
|
# dirList= [dirList[-1]]
|
|
|
|
|
|
if self.online:
|
|
|
currentdate = datetime.datetime.now()
|
|
|
strsplit1 = currentdate.strftime('%Y.%m.%d')
|
|
|
dirList = fnmatch.filter(dirList, strsplit1 + '*')
|
|
|
|
|
|
for thisDir in dirList:
|
|
|
if not os.path.isdir(os.path.join(path, thisDir)):
|
|
|
continue
|
|
|
|
|
|
strsplit = thisDir.split('.')
|
|
|
timeints = [int(i) for i in strsplit]
|
|
|
timelist = datetime.datetime(timeints[0], timeints[1], timeints[2], timeints[3], timeints[4], timeints[5])
|
|
|
utctime = (timelist - datetime.datetime(1970, 1, 1)).total_seconds()
|
|
|
|
|
|
if not self.online:
|
|
|
if (utctime > endsearch2):
|
|
|
continue
|
|
|
|
|
|
if (utctime < startsearch2):
|
|
|
continue
|
|
|
|
|
|
dirListFiltered.append(thisDir)
|
|
|
utclist.append(utctime)
|
|
|
|
|
|
if not dirListFiltered:
|
|
|
print("filtro")
|
|
|
return []
|
|
|
|
|
|
for thisDir in dirListFiltered:
|
|
|
|
|
|
pathFile = os.path.join(self.path, thisDir)
|
|
|
|
|
|
fileList = os.listdir(pathFile)
|
|
|
|
|
|
if not fileList:
|
|
|
continue
|
|
|
|
|
|
for k in range(len(fileList)):
|
|
|
thisFile = str(k) + '.mat'
|
|
|
|
|
|
if not os.path.isfile(os.path.join(pathFile, thisFile)):
|
|
|
continue
|
|
|
|
|
|
fileListFiltered.append(os.path.join(pathFile, thisFile))
|
|
|
|
|
|
return fileListFiltered
|
|
|
|
|
|
def __getNextOnlineFile(self, seconds=40):
|
|
|
|
|
|
filename = self.__getNextOfflineFile()
|
|
|
|
|
|
if filename:
|
|
|
return filename
|
|
|
|
|
|
ncurrentfiles = len(self.fileList)
|
|
|
|
|
|
nTries = 0
|
|
|
while (True):
|
|
|
filelist = self.__findFiles(self.path)
|
|
|
if len(filelist) > ncurrentfiles:
|
|
|
break
|
|
|
|
|
|
nTries += 1
|
|
|
|
|
|
if nTries > 3:
|
|
|
break
|
|
|
|
|
|
print("Waiting %d seconds ..." % seconds)
|
|
|
time.sleep(40)
|
|
|
|
|
|
if not (len(filelist) > ncurrentfiles):
|
|
|
return None
|
|
|
|
|
|
self.fileList = filelist
|
|
|
filename = self.__getNextOfflineFile()
|
|
|
|
|
|
return filename
|
|
|
|
|
|
def __getNextOfflineFile(self):
|
|
|
|
|
|
if self.index >= len(self.fileList):
|
|
|
return None
|
|
|
|
|
|
filename = self.fileList[self.index]
|
|
|
self.index += 1
|
|
|
return filename
|
|
|
|
|
|
def __getNextFile(self):
|
|
|
|
|
|
if self.online:
|
|
|
filename = self.__getNextOnlineFile()
|
|
|
else:
|
|
|
filename = self.__getNextOfflineFile()
|
|
|
return filename
|
|
|
|
|
|
def setup(self, path, startDate=None, endDate=None, startTime=datetime.time(0, 0, 0), endTime=datetime.time(23, 59, 59)):
|
|
|
|
|
|
fileList = self.__findFiles(path, startDate, endDate, startTime, endTime)
|
|
|
|
|
|
if self.online:
|
|
|
self.index = len(fileList) - 1
|
|
|
else:
|
|
|
self.index = 0
|
|
|
|
|
|
self.fileList = fileList
|
|
|
|
|
|
print("fin setup")
|
|
|
|
|
|
def run(self, path=None, startDate=None, endDate=None,
|
|
|
startTime=datetime.time(0, 0, 0),
|
|
|
endTime=datetime.time(23, 59, 59),
|
|
|
walk=True, timezone='ut',
|
|
|
all=0, online=False, ext=None, **kwargs):
|
|
|
|
|
|
self.path = path
|
|
|
self.ext = ext
|
|
|
self.startDate = startDate
|
|
|
self.endDate = endDate
|
|
|
self.startTime = startTime
|
|
|
self.endTime = endTime
|
|
|
self.online = online
|
|
|
self.dataOut.flagNoData = True
|
|
|
|
|
|
if (self.firsttime == True):
|
|
|
self.setup(path, startDate, endDate, startTime, endTime)
|
|
|
self.firsttime = False
|
|
|
|
|
|
|
|
|
if not self.fileList:
|
|
|
self.dataOut.flagNoData = True
|
|
|
print("lista vacia")
|
|
|
return
|
|
|
|
|
|
currentfile = self.__getNextFile()
|
|
|
|
|
|
if not currentfile:
|
|
|
"no file"
|
|
|
return
|
|
|
|
|
|
spc, cspc = self.__readFile(currentfile)
|
|
|
|
|
|
if spc != None:
|
|
|
|
|
|
self.dataOut.data_spc = spc
|
|
|
self.dataOut.data_cspc = cspc
|
|
|
self.dataOut.utctime = self.utcfirst
|
|
|
self.dataOut.flagNoData = False
|
|
|
|
|
|
return 1
|
|
|
|
|
|
|
