##// END OF EJS Templates
schain
RSS
Programa de lectura de datos del Radar Ciri Huancayo
Alexander Valdez -
r593:5bc38e59b2e3
parent child
Show More
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (510 lines changed) Show them Hide them
@@ -9,16 +9,8 from os import listdir
9 9 from os.path import isfile, join
10 10 import datetime
11 11 import cmath
12 from astropy.io.ascii.tests.test_connect import files
13 12
14 # Path needs to be whereever the matlab data files are stored.
15 # This program works by reading how many data folders remain inside of the
16 # path destination. Every iteration of the program it will re-search
17 # the path destination, if new folder(s) are detected, it will read
18 # in all data from say... enough files to cover an hour and generate
19 # spectra and RTI plots.
20
21 class MatReader(ProcessingUnit):
13 class matoffReader(ProcessingUnit):
22 14
23 15 index=None
24 16 list=None
@@ -28,310 +20,246 class MatReader(ProcessingUnit):
28 20 utcmatcounter=0
29 21 utcfirst=None
30 22 utclist=None
31 foldercountercrosscheck=None
32 foldercountercheck=None
33 indexfirsttime=-31
34 23
35 24 def __init__(self):
36 25 self.dataOut = Spectra()
26 self.online = False
37 27 return
38 28
39 # def FIRSTTIMERUNTHROUGH(self,path=None,startDate=None, endDate=None,startTime=datetime.time(0,0,0),
40 # endTime=datetime.time(23,59,59),walk=True,timezone='ut',
41 # all=0,online=True,ext=None,filelist=None, **kwargs):
42 #
43 # foldcountercrosscheck=0
44 # self.foldercountercrosscheck=foldcountercrosscheck
45 #
46 # for g in range(len(filelist)):
47 # strsplit=filelist[g].split('.')
48 # timeints=[int(i) for i in strsplit]
49 # timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
50 # utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
51 # secondlist.append(filelist[g])
52 # self.utclist.append(utctime)
53 #
54 # for k in range(len(secondlist)):
55 #
56 # path1=os.path.join(self.path,secondlist[k])
57 # filecounter=len([name for name in os.listdir(path1)])
58 # # print "Reading from this dir:" +path1
59 # for r in range(filecounter):
60 # matname=str(r)+'.mat'
61 # bork=os.path.join(path1,matname)
62 # # thirdlist is what ends up being the list to all matlab files
63 # thirdlist.append(os.path.join(path1,matname))
64 #
65 # self.utcfirst=utclist[-1]
66 # self.index=-31
67 # self.list=thirdlist
68 #
69 # while (self.index <= -1):
70 # currentfilenewest=self.list[self.index]
71 # print "Reading from this file:" + currentfilenewest
72 # filesplit=currentfilenewest.split("\\")
73 # newsplit=filesplit[-2]
74 # newnewsplit=newsplit.split(".")
75 # newnewsplit=[int(i) for i in newnewsplit]
76 # gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5])
77 # self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds()
78 #
79 #
80 # newsplit=filesplit[-1]
81 # newnewsplit=newsplit.split(".")
82 # goobnum=newnewsplit[0]
83 # goobnum=int(goobnum)
84 #
85 # self.utcfirst=self.utcfirst+goobnum*2
86 # print self.utcfirst
87 #
88 # datastuff=sio.loadmat(currentfilenewest)
89 # dataphase=datastuff.get('phase')
90 # data3=datastuff.get('doppler0')
91 # data4=datastuff.get('doppler1')
92 # data3= np.array(data3)
93 # data4 = np.array(data4)
94 # datacoh=datastuff.get('coherence2')
95 #
96 # datacohphase=datacoh*np.exp(-dataphase*1j)
97 # # data31 = np.fliplr(data3)
98 # # data41 = np.fliplr(data4)
99 #
100 # data31 = data3.reshape((1,data3.shape[0],data3.shape[1]))
101 # data41 = data4.reshape((1,data4.shape[0],data4.shape[1]))
102 # datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1]))
103 #
104 # datastack = np.vstack((data31,data41))
105 #
106 # self.dataOut.pairsList=[(0,1)]
107 # self.dataOut.data_cspc=datacohphase1.copy()
108 # self.dataOut.data_spc = datastack.copy()
109 # self.dataOut.channelList = range(2)
110 # self.dataOut.nProfiles = 25 #this!
111 # self.dataOut.nIncohInt = 1
112 # self.dataOut.nCohInt = 1 #this!
113 # self.dataOut.ippSeconds = 0.004 #this!
114 # self.dataOut.nFFTPoints = 25
115 # self.dataOut.utctime = self.utcfirst
116 # self.dataOut.heightList = np.array(datastuff.get('hts'))
117 #
118 # self.dataOut.flagNoData = False
119 #
120 # self.firsttime=False
121 #
122 # self.index+=1
123 #
124 #
125 # return foldercountercrosscheck, self.firsttime
29 def __setHeader(self, datastuff):
30
31 self.dataOut.pairsList=[(0,1)]
32 self.dataOut.channelList = range(2)
33 self.dataOut.nProfiles = 25 #this!
34 self.dataOut.nIncohInt = 1
35 self.dataOut.nCohInt = 1 #this!
36 self.dataOut.ippSeconds = 0.004 #this!
37 self.dataOut.nFFTPoints = 25
38 self.dataOut.timeZone = 0
39 self.dataOut.heightList = np.array(datastuff.get('hts')).flatten()
126 40
127 def run(self,path=None,startDate=None, endDate=None,startTime=datetime.time(0,0,0),
128 endTime=datetime.time(23,59,59),walk=True,timezone='ut',
129 all=0,online=True,ext=None, **kwargs):
41 def __readFile(self, currentfile):
42 print "Reading from this file:" + currentfile
130 43
131 self.path=path
132 self.ext=ext
133 self.startDate=startDate
134 self.endDate=endDate
135 self.startTime=startTime
136 self.endTime=endTime
137 self.index=0
44 #filesplit=currentfile.split("\\")
45 filesplit=currentfile.split("/")
46 newsplit=filesplit[-2]
47 newnewsplit=newsplit.split(".")
48 newnewsplit=[int(i) for i in newnewsplit]
49 gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5])
50 self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds()
51
52
53 newsplit=filesplit[-1]
54 newnewsplit=newsplit.split(".")
55 goobnum=newnewsplit[0]
56 goobnum=int(goobnum)
57
58 self.utcfirst=self.utcfirst+goobnum*2
59 # if (currentfile[43:]=='0.mat'):
60 # self.utcmatcounter=0
61 # self.utcfirst=self.utclist[self.index]
62
63 # if (self.utcmatcounter>60):
64 # self.utcmatcounter=0
65
66 # print self.utcmatcounter
67 print self.utcfirst
68 try:
69 datastuff=sio.loadmat(currentfile)
70 except:
71 return None, None
72
73 dataphase=datastuff.get('phase')
74 data3=datastuff.get('doppler0')
75 data4=datastuff.get('doppler1')
76 data3= np.array(data3)
77 data4 = np.array(data4)
78 datacoh=datastuff.get('coherence2')
79
80 datacohphase=datacoh*np.exp(-dataphase*1j)
81 # data31 = np.fliplr(data3)
82 # data41 = np.fliplr(data4)
83
84 data31 = data3.reshape((1,data3.shape[0],data3.shape[1]))
85 data41 = data4.reshape((1,data4.shape[0],data4.shape[1]))
86 datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1]))
87
88 datastack = np.vstack((data31,data41))
89
90 self.__setHeader(datastuff)
91
92 spc = datastack
93 cspc = datacohphase1
94
95 return spc, cspc
96
97 def __findFiles(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
138 98
139 filelist = [ f for f in listdir(path)]
140 secondlist=[]
141 thirdlist=[]
99 if startDate == None:
100 startDate = datetime.date(1970,1,1)
101
102 if endDate == None:
103 endDate = datetime.date(2050,1,1)
104
105 startsearch1=datetime.datetime.combine(startDate,startTime)
106 startsearch2=(startsearch1-datetime.datetime(1970,1,1)).total_seconds()
107 endsearch1=datetime.datetime.combine(endDate,endTime)
108 endsearch2=(endsearch1-datetime.datetime(1970,1,1)).total_seconds()
109
110 dirList = listdir(path)
111 dirList = sorted(dirList)
112
113 dirListFiltered=[]
114 fileListFiltered=[]
142 115 utclist=[]
143 foldercountercheck=len([name for name in os.listdir(self.path)])
144 116
145 # ______________________________________________________________________________________________________________________________________________________________-
146 # First time through this acts much like the offline program,
147 # It reads in the latest file in the path to create a list. So, it finds
148 # the very last folder in the path, opens it and reads in the 30 matlab files
149 # inside of that. Then, this loop never runs again.
150
151
152 if (self.firsttime==True):
153 # self.FIRSTTIMERUNTHROUGH(path, self.startDate, self.endDate, self.startTime,
154 # self.endTime, walk, timezone, all, online, ext,filelist)
155 foldercountercrosscheck=0
156 self.utclist=utclist
117 if not dirList:
118 print "No directories found"
119 return []
120
121 for thisDir in dirList:
122 if not os.path.isdir(os.path.join(path, thisDir)):
123 continue
124
125 strsplit=thisDir.split('.')
126 timeints=[int(i) for i in strsplit]
127 timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
128 utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
157 129
130 if not self.online:
131 if (utctime > endsearch2):
132 continue
133
134 if (utctime < startsearch2):
135 continue
158 136
159 for g in range(len(filelist)):
160 strsplit=filelist[g].split('.')
161 timeints=[int(i) for i in strsplit]
162 timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
163 utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
164 secondlist.append(filelist[g])
165 self.utclist.append(utctime)
166
167 for k in range(len(secondlist)):
168
169 path1=os.path.join(self.path,secondlist[k])
170 filecounter=len([name for name in os.listdir(path1)])
171 # print "Reading from this dir:" +path1
172 for r in range(filecounter):
173 matname=str(r)+'.mat'
174 bork=os.path.join(path1,matname)
175 # thirdlist is what ends up being the list to all matlab files
176 thirdlist.append(os.path.join(path1,matname))
137 dirListFiltered.append(thisDir)
138 utclist.append(utctime)
139
140 if not dirListFiltered:
141 print "filtro"
142 return []
143
144 for thisDir in dirListFiltered:
145
146 pathFile = os.path.join(self.path, thisDir)
147
148 fileList = os.listdir(pathFile)
149
150 if not fileList:
151 continue
177 152
178 # Set the index to -31 initially such that it reads the final 30 matlab files in the
179 # path for the first iteration. It gets reset in the continually checking portion.
153 for k in range(len(fileList)):
154 thisFile = str(k)+'.mat'
155
156 if not os.path.isfile(os.path.join(pathFile, thisFile)):
157 continue
158
159 fileListFiltered.append(os.path.join(pathFile, thisFile))
160
161 return fileListFiltered
162
163 def __getNextOnlineFile(self, seconds = 30):
164
165 filename = self.__getNextOfflineFile()
166
167 if filename:
168 return filename
169
170 ncurrentfiles = len(self.fileList)
171
172 nTries = 0
173 while (True):
174 filelist = self.__findFiles(self.path)
175 if len(filelist) > ncurrentfiles:
176 break
180 177
181 self.utcfirst=utclist[-1]
182 self.list=thirdlist
183
184 currentfilenewest=self.list[self.indexfirsttime]
185 print "Reading from this file:" + currentfilenewest
186 filesplit=currentfilenewest.split("\\")
187 newsplit=filesplit[-2]
188 newnewsplit=newsplit.split(".")
189 newnewsplit=[int(i) for i in newnewsplit]
190 gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5])
191 self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds()
192
193
194 newsplit=filesplit[-1]
195 newnewsplit=newsplit.split(".")
196 goobnum=newnewsplit[0]
197 goobnum=int(goobnum)
198
199 self.utcfirst=self.utcfirst+goobnum*2
200 print self.utcfirst
201
202 datastuff=sio.loadmat(currentfilenewest)
203 dataphase=datastuff.get('phase')
204 data3=datastuff.get('doppler0')
205 data4=datastuff.get('doppler1')
206 data3= np.array(data3)
207 data4 = np.array(data4)
208 datacoh=datastuff.get('coherence2')
209
210 datacohphase=datacoh*np.exp(-dataphase*1j)
211 # data31 = np.fliplr(data3)
212 # data41 = np.fliplr(data4)
213
214 data31 = data3.reshape((1,data3.shape[0],data3.shape[1]))
215 data41 = data4.reshape((1,data4.shape[0],data4.shape[1]))
216 datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1]))
217
218 datastack = np.vstack((data31,data41))
219
220 self.dataOut.pairsList=[(0,1)]
221 self.dataOut.data_cspc=datacohphase1.copy()
222 self.dataOut.data_spc = datastack.copy()
223 self.dataOut.channelList = range(2)
224 self.dataOut.nProfiles = 25 #this!
225 self.dataOut.nIncohInt = 1
226 self.dataOut.nCohInt = 1 #this!
227 self.dataOut.ippSeconds = 0.004 #this!
228 self.dataOut.nFFTPoints = 25
229 self.dataOut.utctime = self.utcfirst
230 self.dataOut.heightList = np.array(datastuff.get('hts'))
231
232 self.dataOut.flagNoData = False
233 self.indexfirsttime+=1
178 nTries += 1
234 179
235 if (self.indexfirsttime>=30):
236 self.firsttime=False
237 self.index=0
238
180 if nTries > 3:
181 break
239 182
240 #
183 print "Waiting %d seconds ..." %seconds
184 time.sleep(30)
185
186 if not (len(filelist) > ncurrentfiles):
187 return None
241 188
189 self.fileList = filelist
190 filename = self.__getNextOfflineFile()
242 191
192 return filename
193
194 def __getNextOfflineFile(self):
195
196 if self.index >= len(self.fileList):
197 return None
198
199 filename=self.fileList[self.index]
200 self.index += 1
201 return filename
202
203 def __getNextFile(self):
243 204
244 # __________________________________________________________________________________________________________________________________________________________-
205 if self.online:
206 filename = self.__getNextOnlineFile()
207 else:
208 filename = self.__getNextOfflineFile()
209 return filename
245 210
211 def setup(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
212
213 fileList = self.__findFiles(path, startDate, endDate, startTime, endTime)
214
215 if self.online:
216 self.index = len(fileList) -1
217 else:
218 self.index = 0
246 219
247 # Now we check for new folders, if some are detected, we repeat the process
248 self.firsttime=False
249
250 if(foldercountercheck>foldercountercrosscheck):
251 foldercountercrosscheck=foldercountercheck
220 self.fileList = fileList
221
222 print "fin setup"
223
224 def run(self,path=None,startDate=None, endDate=None,
225 startTime=datetime.time(0,0,0),
226 endTime=datetime.time(23,59,59),
227 walk=True,timezone='ut',
228 all=0,online=False,ext=None,**kwargs):
229
230 self.path=path
231 self.ext=ext
232 self.startDate=startDate
233 self.endDate=endDate
234 self.startTime=startTime
235 self.endTime=endTime
236 self.online = online
237 self.dataOut.flagNoData = True
238
239 if (self.firsttime==True):
240 self.setup(path, startDate, endDate, startTime, endTime)
241 self.firsttime=False
252 242
253
254 for g in range(len(filelist)):
255 strsplit=filelist[g].split('.')
256 timeints=[int(i) for i in strsplit]
257 timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
258 utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
259 secondlist.append(filelist[g])
260 self.utclist.append(utctime)
261
262 for k in range(len(secondlist)):
263
264 path1=os.path.join(self.path,secondlist[k])
265 filecounter=len([name for name in os.listdir(path1)])
266 # print "Reading from this dir:" +path1
267 for r in range(filecounter):
268 matname=str(r)+'.mat'
269 bork=os.path.join(path1,matname)
270 # thirdlist is what ends up being the list to all matlab files
271 thirdlist.append(os.path.join(path1,matname))
243
244 if not self.fileList:
245 self.dataOut.flagNoData = True
246 print "lista vacia"
247 return
248
249 currentfile = self.__getNextFile()
250
251 if not currentfile:
252 "no file"
253 return
254
255 spc, cspc = self.__readFile(currentfile)
256
257 if spc!=None:
272 258
273 # Set the index to -31 initially such that it reads the final 30 matlab files in the
274 # path for the first iteration. It gets reset in the continually checking portion.
275
276 self.utcfirst=utclist[-1]
277 self.list=thirdlist
278
279 currentfilenewest=self.list[self.indexfirsttime]
280 print "Reading from this file:" + currentfilenewest
281 filesplit=currentfilenewest.split("\\")
282 newsplit=filesplit[-2]
283 newnewsplit=newsplit.split(".")
284 newnewsplit=[int(i) for i in newnewsplit]
285 gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5])
286 self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds()
287
288
289 newsplit=filesplit[-1]
290 newnewsplit=newsplit.split(".")
291 goobnum=newnewsplit[0]
292 goobnum=int(goobnum)
293
294 self.utcfirst=self.utcfirst+goobnum*2
295 print "The utc of the file is:" + self.utcfirst
296
297 datastuff=sio.loadmat(currentfilenewest)
298 dataphase=datastuff.get('phase')
299 data3=datastuff.get('doppler0')
300 data4=datastuff.get('doppler1')
301 data3= np.array(data3)
302 data4 = np.array(data4)
303 datacoh=datastuff.get('coherence2')
304
305 datacohphase=datacoh*np.exp(-dataphase*1j)
306 # data31 = np.fliplr(data3)
307 # data41 = np.fliplr(data4)
308
309 data31 = data3.reshape((1,data3.shape[0],data3.shape[1]))
310 data41 = data4.reshape((1,data4.shape[0],data4.shape[1]))
311 datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1]))
312
313 datastack = np.vstack((data31,data41))
314
315 self.dataOut.pairsList=[(0,1)]
316 self.dataOut.data_cspc=datacohphase1.copy()
317 self.dataOut.data_spc = datastack.copy()
318 self.dataOut.channelList = range(2)
319 self.dataOut.nProfiles = 25 #this!
320 self.dataOut.nIncohInt = 1
321 self.dataOut.nCohInt = 1 #this!
322 self.dataOut.ippSeconds = 0.004 #this!
323 self.dataOut.nFFTPoints = 25
259 self.dataOut.data_spc = spc
260 self.dataOut.data_cspc = cspc
324 261 self.dataOut.utctime = self.utcfirst
325 self.dataOut.heightList = np.array(datastuff.get('hts'))
326
327 262 self.dataOut.flagNoData = False
328 self.indexfirsttime+=1
329
330 if (self.indexfirsttime>=30):
331 self.firsttime=False
332 self.index=0
333
334 else:
335 time.sleep(15)
336
337 return 1 No newline at end of file
263
264 return 1
265 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now