##// END OF EJS Templates
schain
RSS
Cambios para la lectura en modo online, ahora en la lectura se filtra por el dia actual, es decir si el folder contiene varios dias, solo leera los folders de la fecha en que nos encontramos
Alexander Valdez -
r598:1b565bb36c44
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,265 +1,273
1 from schainpy.model.data.jrodata import *
1 from schainpy.model.data.jrodata import *
2 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
2 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
3 from schainpy.model.io.jroIO_base import *
3 from schainpy.model.io.jroIO_base import *
4
4
5 import scipy.io as sio
5 import scipy.io as sio
6 import pprint
6 import pprint
7 import numpy as np
7 import numpy as np
8 from os import listdir
8 from os import listdir
9 from os.path import isfile, join
9 from os.path import isfile, join
10 import datetime
10 import datetime
11 import cmath
11 import cmath
12
12
13 class matoffReader(ProcessingUnit):
13 class matoffReader(ProcessingUnit):
14
14
15 index=None
15 index=None
16 list=None
16 list=None
17 firsttime=True
17 firsttime=True
18 utccounter=None
18 utccounter=None
19 utcfiletime=None
19 utcfiletime=None
20 utcmatcounter=0
20 utcmatcounter=0
21 utcfirst=None
21 utcfirst=None
22 utclist=None
22 utclist=None
23
23
24 def __init__(self):
24 def __init__(self):
25 self.dataOut = Spectra()
25 self.dataOut = Spectra()
26 self.online = False
26 self.online = False
27 return
27 return
28
28
29 def __setHeader(self, datastuff):
29 def __setHeader(self, datastuff):
30
30
31 self.dataOut.pairsList=[(0,1)]
31 self.dataOut.pairsList=[(0,1)]
32 self.dataOut.channelList = range(2)
32 self.dataOut.channelList = range(np.array(datastuff.get('power')).shape[1])
33 self.dataOut.nProfiles = 25 #this!
33 self.dataOut.nProfiles = len(np.array(datastuff.get('vel')).flatten()) #this!
34 self.dataOut.nIncohInt = 1
34 self.dataOut.nIncohInt = 20
35 self.dataOut.nCohInt = 1 #this!
35 self.dataOut.nCohInt = 1 #this!
36 self.dataOut.ippSeconds = 0.004 #this!
36 self.dataOut.ippSeconds = 0.004 #this!
37 self.dataOut.nFFTPoints = 25
37 self.dataOut.nFFTPoints = len(np.array(datastuff.get('vel')).flatten())
38 self.dataOut.timeZone = 0
38 self.dataOut.timeZone = 0
39 self.dataOut.heightList = np.array(datastuff.get('hts')).flatten()
39 self.dataOut.heightList = np.array(datastuff.get('hts')).flatten()
40
40
41 def __readFile(self, currentfile):
41 def __readFile(self, currentfile):
42 print "Reading from this file:" + currentfile
42 print "Reading from this file:" + currentfile
43
43
44 #filesplit=currentfile.split("\\")
44 #filesplit=currentfile.split("\\")
45 filesplit=currentfile.split("/")
45 filesplit=currentfile.split("/")
46 newsplit=filesplit[-2]
46 newsplit=filesplit[-2]
47 newnewsplit=newsplit.split(".")
47 newnewsplit=newsplit.split(".")
48 newnewsplit=[int(i) for i in newnewsplit]
48 newnewsplit=[int(i) for i in newnewsplit]
49 gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5])
49 gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5])
50 self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds()
50 self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds()
51
51
52
52
53 newsplit=filesplit[-1]
53 newsplit=filesplit[-1]
54 newnewsplit=newsplit.split(".")
54 newnewsplit=newsplit.split(".")
55 goobnum=newnewsplit[0]
55 goobnum=newnewsplit[0]
56 goobnum=int(goobnum)
56 goobnum=int(goobnum)
57
57
58 self.utcfirst=self.utcfirst+goobnum*2
58 self.utcfirst=self.utcfirst+goobnum*2
59 # if (currentfile[43:]=='0.mat'):
59 # if (currentfile[43:]=='0.mat'):
60 # self.utcmatcounter=0
60 # self.utcmatcounter=0
61 # self.utcfirst=self.utclist[self.index]
61 # self.utcfirst=self.utclist[self.index]
62
62
63 # if (self.utcmatcounter>60):
63 # if (self.utcmatcounter>60):
64 # self.utcmatcounter=0
64 # self.utcmatcounter=0
65
65
66 # print self.utcmatcounter
66 # print self.utcmatcounter
67 print self.utcfirst
67 print self.utcfirst
68 try:
68 try:
69 datastuff=sio.loadmat(currentfile)
69 datastuff=sio.loadmat(currentfile)
70 except:
70 except:
71 return None, None
71 return None, None
72
72
73 dataphase=datastuff.get('phase')
73 dataphase=datastuff.get('phase')
74 data3=datastuff.get('doppler0')
74 data3=datastuff.get('doppler0')
75 data4=datastuff.get('doppler1')
75 data4=datastuff.get('doppler1')
76 data3= np.array(data3)
76 data3= np.array(data3)
77 data4 = np.array(data4)
77 data4 = np.array(data4)
78 datacoh=datastuff.get('coherence2')
78 datacoh=datastuff.get('coherence2')
79
79
80 datacohphase=datacoh*np.exp(-dataphase*1j)
80 datacohphase=datacoh*np.exp(-dataphase*1j)
81 # data31 = np.fliplr(data3)
81 # data31 = np.fliplr(data3)
82 # data41 = np.fliplr(data4)
82 # data41 = np.fliplr(data4)
83
83
84 data31 = data3.reshape((1,data3.shape[0],data3.shape[1]))
84 data31 = data3.reshape((1,data3.shape[0],data3.shape[1]))
85 data41 = data4.reshape((1,data4.shape[0],data4.shape[1]))
85 data41 = data4.reshape((1,data4.shape[0],data4.shape[1]))
86 datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1]))
86 datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1]))
87
87
88 datastack = np.vstack((data31,data41))
88 datastack = np.vstack((data31,data41))
89
89
90 self.__setHeader(datastuff)
90 self.__setHeader(datastuff)
91
91
92 spc = datastack
92 spc = datastack
93 cspc = datacohphase1
93 cspc = datacohphase1
94
94
95 return spc, cspc
95 return spc, cspc
96
96
97 def __findFiles(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
97 def __findFiles(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
98
98
99 if startDate == None:
99 if startDate == None:
100 startDate = datetime.date(1970,1,1)
100 startDate = datetime.date(1970,1,1)
101
101
102 if endDate == None:
102 if endDate == None:
103 endDate = datetime.date(2050,1,1)
103 endDate = datetime.date(2050,1,1)
104
104
105 startsearch1=datetime.datetime.combine(startDate,startTime)
105 startsearch1=datetime.datetime.combine(startDate,startTime)
106 startsearch2=(startsearch1-datetime.datetime(1970,1,1)).total_seconds()
106 startsearch2=(startsearch1-datetime.datetime(1970,1,1)).total_seconds()
107 endsearch1=datetime.datetime.combine(endDate,endTime)
107 endsearch1=datetime.datetime.combine(endDate,endTime)
108 endsearch2=(endsearch1-datetime.datetime(1970,1,1)).total_seconds()
108 endsearch2=(endsearch1-datetime.datetime(1970,1,1)).total_seconds()
109
109
110 dirList = listdir(path)
110 dirList = listdir(path)
111 dirList = sorted(dirList)
111 dirList = sorted(dirList)
112
112
113 dirListFiltered=[]
113 dirListFiltered=[]
114 fileListFiltered=[]
114 fileListFiltered=[]
115 utclist=[]
115 utclist=[]
116
116
117 if not dirList:
117 if not dirList:
118 print "No directories found"
118 print "No directories found"
119 return []
119 return []
120
120
121 #if self.online:
122 # dirList= [dirList[-1]]
123
124 if self.online:
125 currentdate = datetime.datetime.utcnow().date()
126 strsplit1=currentdate.strftime('%Y.%m.%d')
127 dirList = fnmatch.filter(dirList,strsplit1+'*')
128
121 for thisDir in dirList:
129 for thisDir in dirList:
122 if not os.path.isdir(os.path.join(path, thisDir)):
130 if not os.path.isdir(os.path.join(path, thisDir)):
123 continue
131 continue
124
132
125 strsplit=thisDir.split('.')
133 strsplit=thisDir.split('.')
126 timeints=[int(i) for i in strsplit]
134 timeints=[int(i) for i in strsplit]
127 timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
135 timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
128 utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
136 utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
129
137
130 if not self.online:
138 if not self.online:
131 if (utctime > endsearch2):
139 if (utctime > endsearch2):
132 continue
140 continue
133
141
134 if (utctime < startsearch2):
142 if (utctime < startsearch2):
135 continue
143 continue
136
144
137 dirListFiltered.append(thisDir)
145 dirListFiltered.append(thisDir)
138 utclist.append(utctime)
146 utclist.append(utctime)
139
147
140 if not dirListFiltered:
148 if not dirListFiltered:
141 print "filtro"
149 print "filtro"
142 return []
150 return []
143
151
144 for thisDir in dirListFiltered:
152 for thisDir in dirListFiltered:
145
153
146 pathFile = os.path.join(self.path, thisDir)
154 pathFile = os.path.join(self.path, thisDir)
147
155
148 fileList = os.listdir(pathFile)
156 fileList = os.listdir(pathFile)
149
157
150 if not fileList:
158 if not fileList:
151 continue
159 continue
152
160
153 for k in range(len(fileList)):
161 for k in range(len(fileList)):
154 thisFile = str(k)+'.mat'
162 thisFile = str(k)+'.mat'
155
163
156 if not os.path.isfile(os.path.join(pathFile, thisFile)):
164 if not os.path.isfile(os.path.join(pathFile, thisFile)):
157 continue
165 continue
158
166
159 fileListFiltered.append(os.path.join(pathFile, thisFile))
167 fileListFiltered.append(os.path.join(pathFile, thisFile))
160
168
161 return fileListFiltered
169 return fileListFiltered
162
170
163 def __getNextOnlineFile(self, seconds = 30):
171 def __getNextOnlineFile(self, seconds = 40):
164
172
165 filename = self.__getNextOfflineFile()
173 filename = self.__getNextOfflineFile()
166
174
167 if filename:
175 if filename:
168 return filename
176 return filename
169
177
170 ncurrentfiles = len(self.fileList)
178 ncurrentfiles = len(self.fileList)
171
179
172 nTries = 0
180 nTries = 0
173 while (True):
181 while (True):
174 filelist = self.__findFiles(self.path)
182 filelist = self.__findFiles(self.path)
175 if len(filelist) > ncurrentfiles:
183 if len(filelist) > ncurrentfiles:
176 break
184 break
177
185
178 nTries += 1
186 nTries += 1
179
187
180 if nTries > 3:
188 if nTries > 3:
181 break
189 break
182
190
183 print "Waiting %d seconds ..." %seconds
191 print "Waiting %d seconds ..." %seconds
184 time.sleep(30)
192 time.sleep(40)
185
193
186 if not (len(filelist) > ncurrentfiles):
194 if not (len(filelist) > ncurrentfiles):
187 return None
195 return None
188
196
189 self.fileList = filelist
197 self.fileList = filelist
190 filename = self.__getNextOfflineFile()
198 filename = self.__getNextOfflineFile()
191
199
192 return filename
200 return filename
193
201
194 def __getNextOfflineFile(self):
202 def __getNextOfflineFile(self):
195
203
196 if self.index >= len(self.fileList):
204 if self.index >= len(self.fileList):
197 return None
205 return None
198
206
199 filename=self.fileList[self.index]
207 filename=self.fileList[self.index]
200 self.index += 1
208 self.index += 1
201 return filename
209 return filename
202
210
203 def __getNextFile(self):
211 def __getNextFile(self):
204
212
205 if self.online:
213 if self.online:
206 filename = self.__getNextOnlineFile()
214 filename = self.__getNextOnlineFile()
207 else:
215 else:
208 filename = self.__getNextOfflineFile()
216 filename = self.__getNextOfflineFile()
209 return filename
217 return filename
210
218
211 def setup(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
219 def setup(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
212
220
213 fileList = self.__findFiles(path, startDate, endDate, startTime, endTime)
221 fileList = self.__findFiles(path, startDate, endDate, startTime, endTime)
214
222
215 if self.online:
223 if self.online:
216 self.index = len(fileList) -1
224 self.index = len(fileList) -1
217 else:
225 else:
218 self.index = 0
226 self.index = 0
219
227
220 self.fileList = fileList
228 self.fileList = fileList
221
229
222 print "fin setup"
230 print "fin setup"
223
231
224 def run(self,path=None,startDate=None, endDate=None,
232 def run(self,path=None,startDate=None, endDate=None,
225 startTime=datetime.time(0,0,0),
233 startTime=datetime.time(0,0,0),
226 endTime=datetime.time(23,59,59),
234 endTime=datetime.time(23,59,59),
227 walk=True,timezone='ut',
235 walk=True,timezone='ut',
228 all=0,online=False,ext=None,**kwargs):
236 all=0,online=False,ext=None,**kwargs):
229
237
230 self.path=path
238 self.path=path
231 self.ext=ext
239 self.ext=ext
232 self.startDate=startDate
240 self.startDate=startDate
233 self.endDate=endDate
241 self.endDate=endDate
234 self.startTime=startTime
242 self.startTime=startTime
235 self.endTime=endTime
243 self.endTime=endTime
236 self.online = online
244 self.online = online
237 self.dataOut.flagNoData = True
245 self.dataOut.flagNoData = True
238
246
239 if (self.firsttime==True):
247 if (self.firsttime==True):
240 self.setup(path, startDate, endDate, startTime, endTime)
248 self.setup(path, startDate, endDate, startTime, endTime)
241 self.firsttime=False
249 self.firsttime=False
242
250
243
251
244 if not self.fileList:
252 if not self.fileList:
245 self.dataOut.flagNoData = True
253 self.dataOut.flagNoData = True
246 print "lista vacia"
254 print "lista vacia"
247 return
255 return
248
256
249 currentfile = self.__getNextFile()
257 currentfile = self.__getNextFile()
250
258
251 if not currentfile:
259 if not currentfile:
252 "no file"
260 "no file"
253 return
261 return
254
262
255 spc, cspc = self.__readFile(currentfile)
263 spc, cspc = self.__readFile(currentfile)
256
264
257 if spc!=None:
265 if spc!=None:
258
266
259 self.dataOut.data_spc = spc
267 self.dataOut.data_spc = spc
260 self.dataOut.data_cspc = cspc
268 self.dataOut.data_cspc = cspc
261 self.dataOut.utctime = self.utcfirst
269 self.dataOut.utctime = self.utcfirst
262 self.dataOut.flagNoData = False
270 self.dataOut.flagNoData = False
263
271
264 return 1
272 return 1
265 No newline at end of file
273
General Comments 0
You need to be logged in to leave comments. Login now