##// END OF EJS Templates
schain
RSS
Cambios para la lectura en modo online, ahora en la lectura se filtra por el dia actual, es decir si el folder contiene varios dias, solo leera los folders de la fecha en que nos encontramos
Alexander Valdez -
r598:1b565bb36c44
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,265 +1,273
1 1 from schainpy.model.data.jrodata import *
2 2 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
3 3 from schainpy.model.io.jroIO_base import *
4 4
5 5 import scipy.io as sio
6 6 import pprint
7 7 import numpy as np
8 8 from os import listdir
9 9 from os.path import isfile, join
10 10 import datetime
11 11 import cmath
12 12
13 13 class matoffReader(ProcessingUnit):
14 14
15 15 index=None
16 16 list=None
17 17 firsttime=True
18 18 utccounter=None
19 19 utcfiletime=None
20 20 utcmatcounter=0
21 21 utcfirst=None
22 22 utclist=None
23 23
24 24 def __init__(self):
25 25 self.dataOut = Spectra()
26 26 self.online = False
27 27 return
28 28
29 29 def __setHeader(self, datastuff):
30 30
31 31 self.dataOut.pairsList=[(0,1)]
32 self.dataOut.channelList = range(2)
33 self.dataOut.nProfiles = 25 #this!
34 self.dataOut.nIncohInt = 1
32 self.dataOut.channelList = range(np.array(datastuff.get('power')).shape[1])
33 self.dataOut.nProfiles = len(np.array(datastuff.get('vel')).flatten()) #this!
34 self.dataOut.nIncohInt = 20
35 35 self.dataOut.nCohInt = 1 #this!
36 36 self.dataOut.ippSeconds = 0.004 #this!
37 self.dataOut.nFFTPoints = 25
37 self.dataOut.nFFTPoints = len(np.array(datastuff.get('vel')).flatten())
38 38 self.dataOut.timeZone = 0
39 39 self.dataOut.heightList = np.array(datastuff.get('hts')).flatten()
40 40
41 41 def __readFile(self, currentfile):
42 42 print "Reading from this file:" + currentfile
43 43
44 44 #filesplit=currentfile.split("\\")
45 45 filesplit=currentfile.split("/")
46 46 newsplit=filesplit[-2]
47 47 newnewsplit=newsplit.split(".")
48 48 newnewsplit=[int(i) for i in newnewsplit]
49 49 gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5])
50 50 self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds()
51 51
52 52
53 53 newsplit=filesplit[-1]
54 54 newnewsplit=newsplit.split(".")
55 55 goobnum=newnewsplit[0]
56 56 goobnum=int(goobnum)
57 57
58 58 self.utcfirst=self.utcfirst+goobnum*2
59 59 # if (currentfile[43:]=='0.mat'):
60 60 # self.utcmatcounter=0
61 61 # self.utcfirst=self.utclist[self.index]
62 62
63 63 # if (self.utcmatcounter>60):
64 64 # self.utcmatcounter=0
65 65
66 66 # print self.utcmatcounter
67 67 print self.utcfirst
68 68 try:
69 69 datastuff=sio.loadmat(currentfile)
70 70 except:
71 71 return None, None
72 72
73 73 dataphase=datastuff.get('phase')
74 74 data3=datastuff.get('doppler0')
75 75 data4=datastuff.get('doppler1')
76 76 data3= np.array(data3)
77 77 data4 = np.array(data4)
78 78 datacoh=datastuff.get('coherence2')
79 79
80 80 datacohphase=datacoh*np.exp(-dataphase*1j)
81 81 # data31 = np.fliplr(data3)
82 82 # data41 = np.fliplr(data4)
83 83
84 84 data31 = data3.reshape((1,data3.shape[0],data3.shape[1]))
85 85 data41 = data4.reshape((1,data4.shape[0],data4.shape[1]))
86 86 datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1]))
87 87
88 88 datastack = np.vstack((data31,data41))
89 89
90 90 self.__setHeader(datastuff)
91 91
92 92 spc = datastack
93 93 cspc = datacohphase1
94 94
95 95 return spc, cspc
96 96
97 97 def __findFiles(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
98 98
99 99 if startDate == None:
100 100 startDate = datetime.date(1970,1,1)
101 101
102 102 if endDate == None:
103 103 endDate = datetime.date(2050,1,1)
104 104
105 105 startsearch1=datetime.datetime.combine(startDate,startTime)
106 106 startsearch2=(startsearch1-datetime.datetime(1970,1,1)).total_seconds()
107 107 endsearch1=datetime.datetime.combine(endDate,endTime)
108 108 endsearch2=(endsearch1-datetime.datetime(1970,1,1)).total_seconds()
109 109
110 110 dirList = listdir(path)
111 111 dirList = sorted(dirList)
112 112
113 113 dirListFiltered=[]
114 114 fileListFiltered=[]
115 115 utclist=[]
116 116
117 117 if not dirList:
118 118 print "No directories found"
119 119 return []
120 120
121 #if self.online:
122 # dirList= [dirList[-1]]
123
124 if self.online:
125 currentdate = datetime.datetime.utcnow().date()
126 strsplit1=currentdate.strftime('%Y.%m.%d')
127 dirList = fnmatch.filter(dirList,strsplit1+'*')
128
121 129 for thisDir in dirList:
122 130 if not os.path.isdir(os.path.join(path, thisDir)):
123 131 continue
124 132
125 133 strsplit=thisDir.split('.')
126 134 timeints=[int(i) for i in strsplit]
127 135 timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
128 136 utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
129 137
130 138 if not self.online:
131 139 if (utctime > endsearch2):
132 140 continue
133 141
134 142 if (utctime < startsearch2):
135 143 continue
136 144
137 145 dirListFiltered.append(thisDir)
138 146 utclist.append(utctime)
139 147
140 148 if not dirListFiltered:
141 149 print "filtro"
142 150 return []
143 151
144 152 for thisDir in dirListFiltered:
145 153
146 154 pathFile = os.path.join(self.path, thisDir)
147 155
148 156 fileList = os.listdir(pathFile)
149 157
150 158 if not fileList:
151 159 continue
152 160
153 161 for k in range(len(fileList)):
154 162 thisFile = str(k)+'.mat'
155 163
156 164 if not os.path.isfile(os.path.join(pathFile, thisFile)):
157 165 continue
158 166
159 167 fileListFiltered.append(os.path.join(pathFile, thisFile))
160 168
161 169 return fileListFiltered
162 170
163 def __getNextOnlineFile(self, seconds = 30):
171 def __getNextOnlineFile(self, seconds = 40):
164 172
165 173 filename = self.__getNextOfflineFile()
166 174
167 175 if filename:
168 176 return filename
169 177
170 178 ncurrentfiles = len(self.fileList)
171 179
172 180 nTries = 0
173 181 while (True):
174 182 filelist = self.__findFiles(self.path)
175 183 if len(filelist) > ncurrentfiles:
176 184 break
177 185
178 186 nTries += 1
179 187
180 188 if nTries > 3:
181 189 break
182 190
183 191 print "Waiting %d seconds ..." %seconds
184 time.sleep(30)
192 time.sleep(40)
185 193
186 194 if not (len(filelist) > ncurrentfiles):
187 195 return None
188 196
189 197 self.fileList = filelist
190 198 filename = self.__getNextOfflineFile()
191 199
192 200 return filename
193 201
194 202 def __getNextOfflineFile(self):
195 203
196 204 if self.index >= len(self.fileList):
197 205 return None
198 206
199 207 filename=self.fileList[self.index]
200 208 self.index += 1
201 209 return filename
202 210
203 211 def __getNextFile(self):
204 212
205 213 if self.online:
206 214 filename = self.__getNextOnlineFile()
207 215 else:
208 216 filename = self.__getNextOfflineFile()
209 217 return filename
210 218
211 219 def setup(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
212 220
213 221 fileList = self.__findFiles(path, startDate, endDate, startTime, endTime)
214 222
215 223 if self.online:
216 224 self.index = len(fileList) -1
217 225 else:
218 226 self.index = 0
219 227
220 228 self.fileList = fileList
221 229
222 230 print "fin setup"
223 231
224 232 def run(self,path=None,startDate=None, endDate=None,
225 233 startTime=datetime.time(0,0,0),
226 234 endTime=datetime.time(23,59,59),
227 235 walk=True,timezone='ut',
228 236 all=0,online=False,ext=None,**kwargs):
229 237
230 238 self.path=path
231 239 self.ext=ext
232 240 self.startDate=startDate
233 241 self.endDate=endDate
234 242 self.startTime=startTime
235 243 self.endTime=endTime
236 244 self.online = online
237 245 self.dataOut.flagNoData = True
238 246
239 247 if (self.firsttime==True):
240 248 self.setup(path, startDate, endDate, startTime, endTime)
241 249 self.firsttime=False
242 250
243 251
244 252 if not self.fileList:
245 253 self.dataOut.flagNoData = True
246 254 print "lista vacia"
247 255 return
248 256
249 257 currentfile = self.__getNextFile()
250 258
251 259 if not currentfile:
252 260 "no file"
253 261 return
254 262
255 263 spc, cspc = self.__readFile(currentfile)
256 264
257 265 if spc!=None:
258 266
259 267 self.dataOut.data_spc = spc
260 268 self.dataOut.data_cspc = cspc
261 269 self.dataOut.utctime = self.utcfirst
262 270 self.dataOut.flagNoData = False
263 271
264 272 return 1
265 273 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now