##// END OF EJS Templates
schain
RSS
filtro online para tomar datos solo del dia actual
Alexander Valdez -
r602:92234dd29577
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,273 +1,273
1 from schainpy.model.data.jrodata import *
1 from schainpy.model.data.jrodata import *
2 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
2 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
3 from schainpy.model.io.jroIO_base import *
3 from schainpy.model.io.jroIO_base import *
4
4
5 import scipy.io as sio
5 import scipy.io as sio
6 import pprint
6 import pprint
7 import numpy as np
7 import numpy as np
8 from os import listdir
8 from os import listdir
9 from os.path import isfile, join
9 from os.path import isfile, join
10 import datetime
10 import datetime
11 import cmath
11 import cmath
12
12
13 class matoffReader(ProcessingUnit):
13 class matoffReader(ProcessingUnit):
14
14
15 index=None
15 index=None
16 list=None
16 list=None
17 firsttime=True
17 firsttime=True
18 utccounter=None
18 utccounter=None
19 utcfiletime=None
19 utcfiletime=None
20 utcmatcounter=0
20 utcmatcounter=0
21 utcfirst=None
21 utcfirst=None
22 utclist=None
22 utclist=None
23
23
24 def __init__(self):
24 def __init__(self):
25 self.dataOut = Spectra()
25 self.dataOut = Spectra()
26 self.online = False
26 self.online = False
27 return
27 return
28
28
29 def __setHeader(self, datastuff):
29 def __setHeader(self, datastuff):
30
30
31 self.dataOut.pairsList=[(0,1)]
31 self.dataOut.pairsList=[(0,1)]
32 self.dataOut.channelList = range(np.array(datastuff.get('power')).shape[1])
32 self.dataOut.channelList = range(np.array(datastuff.get('power')).shape[1])
33 self.dataOut.nProfiles = len(np.array(datastuff.get('vel')).flatten()) #this!
33 self.dataOut.nProfiles = len(np.array(datastuff.get('vel')).flatten()) #this!
34 self.dataOut.nIncohInt = 20
34 self.dataOut.nIncohInt = 20
35 self.dataOut.nCohInt = 1 #this!
35 self.dataOut.nCohInt = 1 #this!
36 self.dataOut.ippSeconds = 0.004 #this!
36 self.dataOut.ippSeconds = 0.004 #this!
37 self.dataOut.nFFTPoints = len(np.array(datastuff.get('vel')).flatten())
37 self.dataOut.nFFTPoints = len(np.array(datastuff.get('vel')).flatten())
38 self.dataOut.timeZone = 0
38 self.dataOut.timeZone = 0
39 self.dataOut.heightList = np.array(datastuff.get('hts')).flatten()
39 self.dataOut.heightList = np.array(datastuff.get('hts')).flatten()
40
40
41 def __readFile(self, currentfile):
41 def __readFile(self, currentfile):
42 print "Reading from this file:" + currentfile
42 print "Reading from this file:" + currentfile
43
43
44 #filesplit=currentfile.split("\\")
44 #filesplit=currentfile.split("\\")
45 filesplit=currentfile.split("/")
45 filesplit=currentfile.split("/")
46 newsplit=filesplit[-2]
46 newsplit=filesplit[-2]
47 newnewsplit=newsplit.split(".")
47 newnewsplit=newsplit.split(".")
48 newnewsplit=[int(i) for i in newnewsplit]
48 newnewsplit=[int(i) for i in newnewsplit]
49 gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5])
49 gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5])
50 self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds()
50 self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds()
51
51
52
52
53 newsplit=filesplit[-1]
53 newsplit=filesplit[-1]
54 newnewsplit=newsplit.split(".")
54 newnewsplit=newsplit.split(".")
55 goobnum=newnewsplit[0]
55 goobnum=newnewsplit[0]
56 goobnum=int(goobnum)
56 goobnum=int(goobnum)
57
57
58 self.utcfirst=self.utcfirst+goobnum*2
58 self.utcfirst=self.utcfirst+goobnum*2
59 # if (currentfile[43:]=='0.mat'):
59 # if (currentfile[43:]=='0.mat'):
60 # self.utcmatcounter=0
60 # self.utcmatcounter=0
61 # self.utcfirst=self.utclist[self.index]
61 # self.utcfirst=self.utclist[self.index]
62
62
63 # if (self.utcmatcounter>60):
63 # if (self.utcmatcounter>60):
64 # self.utcmatcounter=0
64 # self.utcmatcounter=0
65
65
66 # print self.utcmatcounter
66 # print self.utcmatcounter
67 print self.utcfirst
67 print self.utcfirst
68 try:
68 try:
69 datastuff=sio.loadmat(currentfile)
69 datastuff=sio.loadmat(currentfile)
70 except:
70 except:
71 return None, None
71 return None, None
72
72
73 dataphase=datastuff.get('phase')
73 dataphase=datastuff.get('phase')
74 data3=datastuff.get('doppler0')
74 data3=datastuff.get('doppler0')
75 data4=datastuff.get('doppler1')
75 data4=datastuff.get('doppler1')
76 data3= np.array(data3)
76 data3= np.array(data3)
77 data4 = np.array(data4)
77 data4 = np.array(data4)
78 datacoh=datastuff.get('coherence2')
78 datacoh=datastuff.get('coherence2')
79
79
80 datacohphase=datacoh*np.exp(-dataphase*1j)
80 datacohphase=datacoh*np.exp(-dataphase*1j)
81 # data31 = np.fliplr(data3)
81 # data31 = np.fliplr(data3)
82 # data41 = np.fliplr(data4)
82 # data41 = np.fliplr(data4)
83
83
84 data31 = data3.reshape((1,data3.shape[0],data3.shape[1]))
84 data31 = data3.reshape((1,data3.shape[0],data3.shape[1]))
85 data41 = data4.reshape((1,data4.shape[0],data4.shape[1]))
85 data41 = data4.reshape((1,data4.shape[0],data4.shape[1]))
86 datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1]))
86 datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1]))
87
87
88 datastack = np.vstack((data31,data41))
88 datastack = np.vstack((data31,data41))
89
89
90 self.__setHeader(datastuff)
90 self.__setHeader(datastuff)
91
91
92 spc = datastack
92 spc = datastack
93 cspc = datacohphase1
93 cspc = datacohphase1
94
94
95 return spc, cspc
95 return spc, cspc
96
96
97 def __findFiles(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
97 def __findFiles(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
98
98
99 if startDate == None:
99 if startDate == None:
100 startDate = datetime.date(1970,1,1)
100 startDate = datetime.date(1970,1,1)
101
101
102 if endDate == None:
102 if endDate == None:
103 endDate = datetime.date(2050,1,1)
103 endDate = datetime.date(2050,1,1)
104
104
105 startsearch1=datetime.datetime.combine(startDate,startTime)
105 startsearch1=datetime.datetime.combine(startDate,startTime)
106 startsearch2=(startsearch1-datetime.datetime(1970,1,1)).total_seconds()
106 startsearch2=(startsearch1-datetime.datetime(1970,1,1)).total_seconds()
107 endsearch1=datetime.datetime.combine(endDate,endTime)
107 endsearch1=datetime.datetime.combine(endDate,endTime)
108 endsearch2=(endsearch1-datetime.datetime(1970,1,1)).total_seconds()
108 endsearch2=(endsearch1-datetime.datetime(1970,1,1)).total_seconds()
109
109
110 dirList = listdir(path)
110 dirList = listdir(path)
111 dirList = sorted(dirList)
111 dirList = sorted(dirList)
112
112
113 dirListFiltered=[]
113 dirListFiltered=[]
114 fileListFiltered=[]
114 fileListFiltered=[]
115 utclist=[]
115 utclist=[]
116
116
117 if not dirList:
117 if not dirList:
118 print "No directories found"
118 print "No directories found"
119 return []
119 return []
120
120
121 #if self.online:
121 #if self.online:
122 # dirList= [dirList[-1]]
122 # dirList= [dirList[-1]]
123
123
124 if self.online:
124 if self.online:
125 currentdate = datetime.datetime.utcnow().date()
125 currentdate = datetime.datetime.now()
126 strsplit1=currentdate.strftime('%Y.%m.%d')
126 strsplit1=currentdate.strftime('%Y.%m.%d')
127 dirList = fnmatch.filter(dirList,strsplit1+'*')
127 dirList = fnmatch.filter(dirList,strsplit1+'*')
128
128
129 for thisDir in dirList:
129 for thisDir in dirList:
130 if not os.path.isdir(os.path.join(path, thisDir)):
130 if not os.path.isdir(os.path.join(path, thisDir)):
131 continue
131 continue
132
132
133 strsplit=thisDir.split('.')
133 strsplit=thisDir.split('.')
134 timeints=[int(i) for i in strsplit]
134 timeints=[int(i) for i in strsplit]
135 timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
135 timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
136 utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
136 utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
137
137
138 if not self.online:
138 if not self.online:
139 if (utctime > endsearch2):
139 if (utctime > endsearch2):
140 continue
140 continue
141
141
142 if (utctime < startsearch2):
142 if (utctime < startsearch2):
143 continue
143 continue
144
144
145 dirListFiltered.append(thisDir)
145 dirListFiltered.append(thisDir)
146 utclist.append(utctime)
146 utclist.append(utctime)
147
147
148 if not dirListFiltered:
148 if not dirListFiltered:
149 print "filtro"
149 print "filtro"
150 return []
150 return []
151
151
152 for thisDir in dirListFiltered:
152 for thisDir in dirListFiltered:
153
153
154 pathFile = os.path.join(self.path, thisDir)
154 pathFile = os.path.join(self.path, thisDir)
155
155
156 fileList = os.listdir(pathFile)
156 fileList = os.listdir(pathFile)
157
157
158 if not fileList:
158 if not fileList:
159 continue
159 continue
160
160
161 for k in range(len(fileList)):
161 for k in range(len(fileList)):
162 thisFile = str(k)+'.mat'
162 thisFile = str(k)+'.mat'
163
163
164 if not os.path.isfile(os.path.join(pathFile, thisFile)):
164 if not os.path.isfile(os.path.join(pathFile, thisFile)):
165 continue
165 continue
166
166
167 fileListFiltered.append(os.path.join(pathFile, thisFile))
167 fileListFiltered.append(os.path.join(pathFile, thisFile))
168
168
169 return fileListFiltered
169 return fileListFiltered
170
170
171 def __getNextOnlineFile(self, seconds = 40):
171 def __getNextOnlineFile(self, seconds = 40):
172
172
173 filename = self.__getNextOfflineFile()
173 filename = self.__getNextOfflineFile()
174
174
175 if filename:
175 if filename:
176 return filename
176 return filename
177
177
178 ncurrentfiles = len(self.fileList)
178 ncurrentfiles = len(self.fileList)
179
179
180 nTries = 0
180 nTries = 0
181 while (True):
181 while (True):
182 filelist = self.__findFiles(self.path)
182 filelist = self.__findFiles(self.path)
183 if len(filelist) > ncurrentfiles:
183 if len(filelist) > ncurrentfiles:
184 break
184 break
185
185
186 nTries += 1
186 nTries += 1
187
187
188 if nTries > 3:
188 if nTries > 3:
189 break
189 break
190
190
191 print "Waiting %d seconds ..." %seconds
191 print "Waiting %d seconds ..." %seconds
192 time.sleep(40)
192 time.sleep(40)
193
193
194 if not (len(filelist) > ncurrentfiles):
194 if not (len(filelist) > ncurrentfiles):
195 return None
195 return None
196
196
197 self.fileList = filelist
197 self.fileList = filelist
198 filename = self.__getNextOfflineFile()
198 filename = self.__getNextOfflineFile()
199
199
200 return filename
200 return filename
201
201
202 def __getNextOfflineFile(self):
202 def __getNextOfflineFile(self):
203
203
204 if self.index >= len(self.fileList):
204 if self.index >= len(self.fileList):
205 return None
205 return None
206
206
207 filename=self.fileList[self.index]
207 filename=self.fileList[self.index]
208 self.index += 1
208 self.index += 1
209 return filename
209 return filename
210
210
211 def __getNextFile(self):
211 def __getNextFile(self):
212
212
213 if self.online:
213 if self.online:
214 filename = self.__getNextOnlineFile()
214 filename = self.__getNextOnlineFile()
215 else:
215 else:
216 filename = self.__getNextOfflineFile()
216 filename = self.__getNextOfflineFile()
217 return filename
217 return filename
218
218
219 def setup(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
219 def setup(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
220
220
221 fileList = self.__findFiles(path, startDate, endDate, startTime, endTime)
221 fileList = self.__findFiles(path, startDate, endDate, startTime, endTime)
222
222
223 if self.online:
223 if self.online:
224 self.index = len(fileList) -1
224 self.index = len(fileList) -1
225 else:
225 else:
226 self.index = 0
226 self.index = 0
227
227
228 self.fileList = fileList
228 self.fileList = fileList
229
229
230 print "fin setup"
230 print "fin setup"
231
231
232 def run(self,path=None,startDate=None, endDate=None,
232 def run(self,path=None,startDate=None, endDate=None,
233 startTime=datetime.time(0,0,0),
233 startTime=datetime.time(0,0,0),
234 endTime=datetime.time(23,59,59),
234 endTime=datetime.time(23,59,59),
235 walk=True,timezone='ut',
235 walk=True,timezone='ut',
236 all=0,online=False,ext=None,**kwargs):
236 all=0,online=False,ext=None,**kwargs):
237
237
238 self.path=path
238 self.path=path
239 self.ext=ext
239 self.ext=ext
240 self.startDate=startDate
240 self.startDate=startDate
241 self.endDate=endDate
241 self.endDate=endDate
242 self.startTime=startTime
242 self.startTime=startTime
243 self.endTime=endTime
243 self.endTime=endTime
244 self.online = online
244 self.online = online
245 self.dataOut.flagNoData = True
245 self.dataOut.flagNoData = True
246
246
247 if (self.firsttime==True):
247 if (self.firsttime==True):
248 self.setup(path, startDate, endDate, startTime, endTime)
248 self.setup(path, startDate, endDate, startTime, endTime)
249 self.firsttime=False
249 self.firsttime=False
250
250
251
251
252 if not self.fileList:
252 if not self.fileList:
253 self.dataOut.flagNoData = True
253 self.dataOut.flagNoData = True
254 print "lista vacia"
254 print "lista vacia"
255 return
255 return
256
256
257 currentfile = self.__getNextFile()
257 currentfile = self.__getNextFile()
258
258
259 if not currentfile:
259 if not currentfile:
260 "no file"
260 "no file"
261 return
261 return
262
262
263 spc, cspc = self.__readFile(currentfile)
263 spc, cspc = self.__readFile(currentfile)
264
264
265 if spc!=None:
265 if spc!=None:
266
266
267 self.dataOut.data_spc = spc
267 self.dataOut.data_spc = spc
268 self.dataOut.data_cspc = cspc
268 self.dataOut.data_cspc = cspc
269 self.dataOut.utctime = self.utcfirst
269 self.dataOut.utctime = self.utcfirst
270 self.dataOut.flagNoData = False
270 self.dataOut.flagNoData = False
271
271
272 return 1
272 return 1
273 No newline at end of file
273
General Comments 0
You need to be logged in to leave comments. Login now