jroIO_matlab.py
272 lines
| 8.2 KiB
| text/x-python
|
PythonLexer
Alexander Valdez
|
r591 | from schainpy.model.data.jrodata import * | |
| from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation | |||
| from schainpy.model.io.jroIO_base import * | |||
| import scipy.io as sio | |||
| import pprint | |||
| import numpy as np | |||
| from os import listdir | |||
| from os.path import isfile, join | |||
| import datetime | |||
| import cmath | |||
|
Alexander Valdez
|
r593 | class matoffReader(ProcessingUnit): | |
|
Alexander Valdez
|
r591 | ||
| index=None | |||
| list=None | |||
| firsttime=True | |||
| utccounter=None | |||
| utcfiletime=None | |||
| utcmatcounter=0 | |||
| utcfirst=None | |||
| utclist=None | |||
| def __init__(self): | |||
| self.dataOut = Spectra() | |||
|
Alexander Valdez
|
r593 | self.online = False | |
|
Alexander Valdez
|
r591 | return | |
|
Alexander Valdez
|
r593 | def __setHeader(self, datastuff): | |
| self.dataOut.pairsList=[(0,1)] | |||
|
Alexander Valdez
|
r598 | self.dataOut.channelList = range(np.array(datastuff.get('power')).shape[1]) | |
| self.dataOut.nProfiles = len(np.array(datastuff.get('vel')).flatten()) #this! | |||
| self.dataOut.nIncohInt = 20 | |||
|
Alexander Valdez
|
r593 | self.dataOut.nCohInt = 1 #this! | |
| self.dataOut.ippSeconds = 0.004 #this! | |||
|
Alexander Valdez
|
r598 | self.dataOut.nFFTPoints = len(np.array(datastuff.get('vel')).flatten()) | |
|
Alexander Valdez
|
r593 | self.dataOut.timeZone = 0 | |
| self.dataOut.heightList = np.array(datastuff.get('hts')).flatten() | |||
|
Alexander Valdez
|
r591 | ||
|
Alexander Valdez
|
r593 | def __readFile(self, currentfile): | |
| print "Reading from this file:" + currentfile | |||
|
Alexander Valdez
|
r591 | ||
|
Alexander Valdez
|
r593 | #filesplit=currentfile.split("\\") | |
| filesplit=currentfile.split("/") | |||
| newsplit=filesplit[-2] | |||
| newnewsplit=newsplit.split(".") | |||
| newnewsplit=[int(i) for i in newnewsplit] | |||
| gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5]) | |||
| self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds() | |||
| newsplit=filesplit[-1] | |||
| newnewsplit=newsplit.split(".") | |||
| goobnum=newnewsplit[0] | |||
| goobnum=int(goobnum) | |||
| self.utcfirst=self.utcfirst+goobnum*2 | |||
| # if (currentfile[43:]=='0.mat'): | |||
| # self.utcmatcounter=0 | |||
| # self.utcfirst=self.utclist[self.index] | |||
| # if (self.utcmatcounter>60): | |||
| # self.utcmatcounter=0 | |||
| # print self.utcmatcounter | |||
| print self.utcfirst | |||
| try: | |||
| datastuff=sio.loadmat(currentfile) | |||
| except: | |||
| return None, None | |||
| dataphase=datastuff.get('phase') | |||
| data3=datastuff.get('doppler0') | |||
| data4=datastuff.get('doppler1') | |||
| data3= np.array(data3) | |||
| data4 = np.array(data4) | |||
| datacoh=datastuff.get('coherence2') | |||
| datacohphase=datacoh*np.exp(-dataphase*1j) | |||
| # data31 = np.fliplr(data3) | |||
| # data41 = np.fliplr(data4) | |||
| data31 = data3.reshape((1,data3.shape[0],data3.shape[1])) | |||
| data41 = data4.reshape((1,data4.shape[0],data4.shape[1])) | |||
| datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1])) | |||
| datastack = np.vstack((data31,data41)) | |||
| self.__setHeader(datastuff) | |||
| spc = datastack | |||
| cspc = datacohphase1 | |||
| return spc, cspc | |||
| def __findFiles(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)): | |||
|
Alexander Valdez
|
r591 | ||
|
Alexander Valdez
|
r593 | if startDate == None: | |
| startDate = datetime.date(1970,1,1) | |||
| if endDate == None: | |||
| endDate = datetime.date(2050,1,1) | |||
| startsearch1=datetime.datetime.combine(startDate,startTime) | |||
| startsearch2=(startsearch1-datetime.datetime(1970,1,1)).total_seconds() | |||
| endsearch1=datetime.datetime.combine(endDate,endTime) | |||
| endsearch2=(endsearch1-datetime.datetime(1970,1,1)).total_seconds() | |||
| dirList = listdir(path) | |||
| dirList = sorted(dirList) | |||
| dirListFiltered=[] | |||
| fileListFiltered=[] | |||
|
Alexander Valdez
|
r591 | utclist=[] | |
|
Alexander Valdez
|
r593 | if not dirList: | |
| print "No directories found" | |||
| return [] | |||
|
Alexander Valdez
|
r598 | #if self.online: | |
| # dirList= [dirList[-1]] | |||
| if self.online: | |||
|
Alexander Valdez
|
r602 | currentdate = datetime.datetime.now() | |
|
Alexander Valdez
|
r598 | strsplit1=currentdate.strftime('%Y.%m.%d') | |
| dirList = fnmatch.filter(dirList,strsplit1+'*') | |||
|
Alexander Valdez
|
r593 | for thisDir in dirList: | |
| if not os.path.isdir(os.path.join(path, thisDir)): | |||
| continue | |||
| strsplit=thisDir.split('.') | |||
| timeints=[int(i) for i in strsplit] | |||
| timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5]) | |||
| utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds() | |||
|
Alexander Valdez
|
r591 | ||
|
Alexander Valdez
|
r593 | if not self.online: | |
| if (utctime > endsearch2): | |||
| continue | |||
| if (utctime < startsearch2): | |||
| continue | |||
|
Alexander Valdez
|
r591 | ||
|
Alexander Valdez
|
r593 | dirListFiltered.append(thisDir) | |
| utclist.append(utctime) | |||
| if not dirListFiltered: | |||
| print "filtro" | |||
| return [] | |||
| for thisDir in dirListFiltered: | |||
| pathFile = os.path.join(self.path, thisDir) | |||
| fileList = os.listdir(pathFile) | |||
| if not fileList: | |||
| continue | |||
|
Alexander Valdez
|
r591 | ||
|
Alexander Valdez
|
r593 | for k in range(len(fileList)): | |
| thisFile = str(k)+'.mat' | |||
| if not os.path.isfile(os.path.join(pathFile, thisFile)): | |||
| continue | |||
| fileListFiltered.append(os.path.join(pathFile, thisFile)) | |||
| return fileListFiltered | |||
|
Alexander Valdez
|
r598 | def __getNextOnlineFile(self, seconds = 40): | |
|
Alexander Valdez
|
r593 | ||
| filename = self.__getNextOfflineFile() | |||
| if filename: | |||
| return filename | |||
| ncurrentfiles = len(self.fileList) | |||
| nTries = 0 | |||
| while (True): | |||
| filelist = self.__findFiles(self.path) | |||
| if len(filelist) > ncurrentfiles: | |||
| break | |||
|
Alexander Valdez
|
r591 | ||
|
Alexander Valdez
|
r593 | nTries += 1 | |
|
Alexander Valdez
|
r591 | ||
|
Alexander Valdez
|
r593 | if nTries > 3: | |
| break | |||
|
Alexander Valdez
|
r591 | ||
|
Alexander Valdez
|
r593 | print "Waiting %d seconds ..." %seconds | |
|
Alexander Valdez
|
r598 | time.sleep(40) | |
|
Alexander Valdez
|
r593 | ||
| if not (len(filelist) > ncurrentfiles): | |||
| return None | |||
|
Alexander Valdez
|
r591 | ||
|
Alexander Valdez
|
r593 | self.fileList = filelist | |
| filename = self.__getNextOfflineFile() | |||
|
Alexander Valdez
|
r591 | ||
|
Alexander Valdez
|
r593 | return filename | |
| def __getNextOfflineFile(self): | |||
| if self.index >= len(self.fileList): | |||
| return None | |||
| filename=self.fileList[self.index] | |||
| self.index += 1 | |||
| return filename | |||
| def __getNextFile(self): | |||
|
Alexander Valdez
|
r591 | ||
|
Alexander Valdez
|
r593 | if self.online: | |
| filename = self.__getNextOnlineFile() | |||
| else: | |||
| filename = self.__getNextOfflineFile() | |||
| return filename | |||
|
Alexander Valdez
|
r591 | ||
|
Alexander Valdez
|
r593 | def setup(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)): | |
| fileList = self.__findFiles(path, startDate, endDate, startTime, endTime) | |||
| if self.online: | |||
| self.index = len(fileList) -1 | |||
| else: | |||
| self.index = 0 | |||
|
Alexander Valdez
|
r591 | ||
|
Alexander Valdez
|
r593 | self.fileList = fileList | |
| print "fin setup" | |||
| def run(self,path=None,startDate=None, endDate=None, | |||
| startTime=datetime.time(0,0,0), | |||
| endTime=datetime.time(23,59,59), | |||
| walk=True,timezone='ut', | |||
| all=0,online=False,ext=None,**kwargs): | |||
| self.path=path | |||
| self.ext=ext | |||
| self.startDate=startDate | |||
| self.endDate=endDate | |||
| self.startTime=startTime | |||
| self.endTime=endTime | |||
| self.online = online | |||
| self.dataOut.flagNoData = True | |||
| if (self.firsttime==True): | |||
| self.setup(path, startDate, endDate, startTime, endTime) | |||
| self.firsttime=False | |||
|
Alexander Valdez
|
r591 | ||
|
Alexander Valdez
|
r593 | ||
| if not self.fileList: | |||
| self.dataOut.flagNoData = True | |||
| print "lista vacia" | |||
| return | |||
| currentfile = self.__getNextFile() | |||
| if not currentfile: | |||
| "no file" | |||
| return | |||
| spc, cspc = self.__readFile(currentfile) | |||
| if spc!=None: | |||
|
Alexander Valdez
|
r591 | ||
|
Alexander Valdez
|
r593 | self.dataOut.data_spc = spc | |
| self.dataOut.data_cspc = cspc | |||
|
Alexander Valdez
|
r591 | self.dataOut.utctime = self.utcfirst | |
| self.dataOut.flagNoData = False | |||
|
Alexander Valdez
|
r593 | ||
| return 1 | |||