##// END OF EJS Templates
schain-jc
RSS
filtro online para tomar datos solo del dia actual
Alexander Valdez -
r602:92234dd29577
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,273 +1,273
1 1 from schainpy.model.data.jrodata import *
2 2 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
3 3 from schainpy.model.io.jroIO_base import *
4 4
5 5 import scipy.io as sio
6 6 import pprint
7 7 import numpy as np
8 8 from os import listdir
9 9 from os.path import isfile, join
10 10 import datetime
11 11 import cmath
12 12
13 13 class matoffReader(ProcessingUnit):
14 14
15 15 index=None
16 16 list=None
17 17 firsttime=True
18 18 utccounter=None
19 19 utcfiletime=None
20 20 utcmatcounter=0
21 21 utcfirst=None
22 22 utclist=None
23 23
24 24 def __init__(self):
25 25 self.dataOut = Spectra()
26 26 self.online = False
27 27 return
28 28
29 29 def __setHeader(self, datastuff):
30 30
31 31 self.dataOut.pairsList=[(0,1)]
32 32 self.dataOut.channelList = range(np.array(datastuff.get('power')).shape[1])
33 33 self.dataOut.nProfiles = len(np.array(datastuff.get('vel')).flatten()) #this!
34 34 self.dataOut.nIncohInt = 20
35 35 self.dataOut.nCohInt = 1 #this!
36 36 self.dataOut.ippSeconds = 0.004 #this!
37 37 self.dataOut.nFFTPoints = len(np.array(datastuff.get('vel')).flatten())
38 38 self.dataOut.timeZone = 0
39 39 self.dataOut.heightList = np.array(datastuff.get('hts')).flatten()
40 40
41 41 def __readFile(self, currentfile):
42 42 print "Reading from this file:" + currentfile
43 43
44 44 #filesplit=currentfile.split("\\")
45 45 filesplit=currentfile.split("/")
46 46 newsplit=filesplit[-2]
47 47 newnewsplit=newsplit.split(".")
48 48 newnewsplit=[int(i) for i in newnewsplit]
49 49 gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5])
50 50 self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds()
51 51
52 52
53 53 newsplit=filesplit[-1]
54 54 newnewsplit=newsplit.split(".")
55 55 goobnum=newnewsplit[0]
56 56 goobnum=int(goobnum)
57 57
58 58 self.utcfirst=self.utcfirst+goobnum*2
59 59 # if (currentfile[43:]=='0.mat'):
60 60 # self.utcmatcounter=0
61 61 # self.utcfirst=self.utclist[self.index]
62 62
63 63 # if (self.utcmatcounter>60):
64 64 # self.utcmatcounter=0
65 65
66 66 # print self.utcmatcounter
67 67 print self.utcfirst
68 68 try:
69 69 datastuff=sio.loadmat(currentfile)
70 70 except:
71 71 return None, None
72 72
73 73 dataphase=datastuff.get('phase')
74 74 data3=datastuff.get('doppler0')
75 75 data4=datastuff.get('doppler1')
76 76 data3= np.array(data3)
77 77 data4 = np.array(data4)
78 78 datacoh=datastuff.get('coherence2')
79 79
80 80 datacohphase=datacoh*np.exp(-dataphase*1j)
81 81 # data31 = np.fliplr(data3)
82 82 # data41 = np.fliplr(data4)
83 83
84 84 data31 = data3.reshape((1,data3.shape[0],data3.shape[1]))
85 85 data41 = data4.reshape((1,data4.shape[0],data4.shape[1]))
86 86 datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1]))
87 87
88 88 datastack = np.vstack((data31,data41))
89 89
90 90 self.__setHeader(datastuff)
91 91
92 92 spc = datastack
93 93 cspc = datacohphase1
94 94
95 95 return spc, cspc
96 96
97 97 def __findFiles(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
98 98
99 99 if startDate == None:
100 100 startDate = datetime.date(1970,1,1)
101 101
102 102 if endDate == None:
103 103 endDate = datetime.date(2050,1,1)
104 104
105 105 startsearch1=datetime.datetime.combine(startDate,startTime)
106 106 startsearch2=(startsearch1-datetime.datetime(1970,1,1)).total_seconds()
107 107 endsearch1=datetime.datetime.combine(endDate,endTime)
108 108 endsearch2=(endsearch1-datetime.datetime(1970,1,1)).total_seconds()
109 109
110 110 dirList = listdir(path)
111 111 dirList = sorted(dirList)
112 112
113 113 dirListFiltered=[]
114 114 fileListFiltered=[]
115 115 utclist=[]
116 116
117 117 if not dirList:
118 118 print "No directories found"
119 119 return []
120 120
121 121 #if self.online:
122 122 # dirList= [dirList[-1]]
123 123
124 124 if self.online:
125 currentdate = datetime.datetime.utcnow().date()
125 currentdate = datetime.datetime.now()
126 126 strsplit1=currentdate.strftime('%Y.%m.%d')
127 127 dirList = fnmatch.filter(dirList,strsplit1+'*')
128 128
129 129 for thisDir in dirList:
130 130 if not os.path.isdir(os.path.join(path, thisDir)):
131 131 continue
132 132
133 133 strsplit=thisDir.split('.')
134 134 timeints=[int(i) for i in strsplit]
135 135 timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
136 136 utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
137 137
138 138 if not self.online:
139 139 if (utctime > endsearch2):
140 140 continue
141 141
142 142 if (utctime < startsearch2):
143 143 continue
144 144
145 145 dirListFiltered.append(thisDir)
146 146 utclist.append(utctime)
147 147
148 148 if not dirListFiltered:
149 149 print "filtro"
150 150 return []
151 151
152 152 for thisDir in dirListFiltered:
153 153
154 154 pathFile = os.path.join(self.path, thisDir)
155 155
156 156 fileList = os.listdir(pathFile)
157 157
158 158 if not fileList:
159 159 continue
160 160
161 161 for k in range(len(fileList)):
162 162 thisFile = str(k)+'.mat'
163 163
164 164 if not os.path.isfile(os.path.join(pathFile, thisFile)):
165 165 continue
166 166
167 167 fileListFiltered.append(os.path.join(pathFile, thisFile))
168 168
169 169 return fileListFiltered
170 170
171 171 def __getNextOnlineFile(self, seconds = 40):
172 172
173 173 filename = self.__getNextOfflineFile()
174 174
175 175 if filename:
176 176 return filename
177 177
178 178 ncurrentfiles = len(self.fileList)
179 179
180 180 nTries = 0
181 181 while (True):
182 182 filelist = self.__findFiles(self.path)
183 183 if len(filelist) > ncurrentfiles:
184 184 break
185 185
186 186 nTries += 1
187 187
188 188 if nTries > 3:
189 189 break
190 190
191 191 print "Waiting %d seconds ..." %seconds
192 192 time.sleep(40)
193 193
194 194 if not (len(filelist) > ncurrentfiles):
195 195 return None
196 196
197 197 self.fileList = filelist
198 198 filename = self.__getNextOfflineFile()
199 199
200 200 return filename
201 201
202 202 def __getNextOfflineFile(self):
203 203
204 204 if self.index >= len(self.fileList):
205 205 return None
206 206
207 207 filename=self.fileList[self.index]
208 208 self.index += 1
209 209 return filename
210 210
211 211 def __getNextFile(self):
212 212
213 213 if self.online:
214 214 filename = self.__getNextOnlineFile()
215 215 else:
216 216 filename = self.__getNextOfflineFile()
217 217 return filename
218 218
219 219 def setup(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
220 220
221 221 fileList = self.__findFiles(path, startDate, endDate, startTime, endTime)
222 222
223 223 if self.online:
224 224 self.index = len(fileList) -1
225 225 else:
226 226 self.index = 0
227 227
228 228 self.fileList = fileList
229 229
230 230 print "fin setup"
231 231
232 232 def run(self,path=None,startDate=None, endDate=None,
233 233 startTime=datetime.time(0,0,0),
234 234 endTime=datetime.time(23,59,59),
235 235 walk=True,timezone='ut',
236 236 all=0,online=False,ext=None,**kwargs):
237 237
238 238 self.path=path
239 239 self.ext=ext
240 240 self.startDate=startDate
241 241 self.endDate=endDate
242 242 self.startTime=startTime
243 243 self.endTime=endTime
244 244 self.online = online
245 245 self.dataOut.flagNoData = True
246 246
247 247 if (self.firsttime==True):
248 248 self.setup(path, startDate, endDate, startTime, endTime)
249 249 self.firsttime=False
250 250
251 251
252 252 if not self.fileList:
253 253 self.dataOut.flagNoData = True
254 254 print "lista vacia"
255 255 return
256 256
257 257 currentfile = self.__getNextFile()
258 258
259 259 if not currentfile:
260 260 "no file"
261 261 return
262 262
263 263 spc, cspc = self.__readFile(currentfile)
264 264
265 265 if spc!=None:
266 266
267 267 self.dataOut.data_spc = spc
268 268 self.dataOut.data_cspc = cspc
269 269 self.dataOut.utctime = self.utcfirst
270 270 self.dataOut.flagNoData = False
271 271
272 272 return 1
273 273 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now