##// END OF EJS Templates
schain
RSS
Programa de lectura de datos del Radar Ciri Huancayo
Alexander Valdez -
r593:5bc38e59b2e3
parent child
Show More
Show file before | Show file after | Hide comments
@@ -9,16 +9,8 from os import listdir
9 from os.path import isfile, join
9 from os.path import isfile, join
10 import datetime
10 import datetime
11 import cmath
11 import cmath
12 from astropy.io.ascii.tests.test_connect import files
13
12
14 # Path needs to be whereever the matlab data files are stored.
13 class matoffReader(ProcessingUnit):
15 # This program works by reading how many data folders remain inside of the
16 # path destination. Every iteration of the program it will re-search
17 # the path destination, if new folder(s) are detected, it will read
18 # in all data from say... enough files to cover an hour and generate
19 # spectra and RTI plots.
20
21 class MatReader(ProcessingUnit):
22
14
23 index=None
15 index=None
24 list=None
16 list=None
@@ -28,162 +20,29 class MatReader(ProcessingUnit):
28 utcmatcounter=0
20 utcmatcounter=0
29 utcfirst=None
21 utcfirst=None
30 utclist=None
22 utclist=None
31 foldercountercrosscheck=None
32 foldercountercheck=None
33 indexfirsttime=-31
34
23
35 def __init__(self):
24 def __init__(self):
36 self.dataOut = Spectra()
25 self.dataOut = Spectra()
26 self.online = False
37 return
27 return
38
28
39 # def FIRSTTIMERUNTHROUGH(self,path=None,startDate=None, endDate=None,startTime=datetime.time(0,0,0),
29 def __setHeader(self, datastuff):
40 # endTime=datetime.time(23,59,59),walk=True,timezone='ut',
41 # all=0,online=True,ext=None,filelist=None, **kwargs):
42 #
43 # foldcountercrosscheck=0
44 # self.foldercountercrosscheck=foldcountercrosscheck
45 #
46 # for g in range(len(filelist)):
47 # strsplit=filelist[g].split('.')
48 # timeints=[int(i) for i in strsplit]
49 # timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
50 # utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
51 # secondlist.append(filelist[g])
52 # self.utclist.append(utctime)
53 #
54 # for k in range(len(secondlist)):
55 #
56 # path1=os.path.join(self.path,secondlist[k])
57 # filecounter=len([name for name in os.listdir(path1)])
58 # # print "Reading from this dir:" +path1
59 # for r in range(filecounter):
60 # matname=str(r)+'.mat'
61 # bork=os.path.join(path1,matname)
62 # # thirdlist is what ends up being the list to all matlab files
63 # thirdlist.append(os.path.join(path1,matname))
64 #
65 # self.utcfirst=utclist[-1]
66 # self.index=-31
67 # self.list=thirdlist
68 #
69 # while (self.index <= -1):
70 # currentfilenewest=self.list[self.index]
71 # print "Reading from this file:" + currentfilenewest
72 # filesplit=currentfilenewest.split("\\")
73 # newsplit=filesplit[-2]
74 # newnewsplit=newsplit.split(".")
75 # newnewsplit=[int(i) for i in newnewsplit]
76 # gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5])
77 # self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds()
78 #
79 #
80 # newsplit=filesplit[-1]
81 # newnewsplit=newsplit.split(".")
82 # goobnum=newnewsplit[0]
83 # goobnum=int(goobnum)
84 #
85 # self.utcfirst=self.utcfirst+goobnum*2
86 # print self.utcfirst
87 #
88 # datastuff=sio.loadmat(currentfilenewest)
89 # dataphase=datastuff.get('phase')
90 # data3=datastuff.get('doppler0')
91 # data4=datastuff.get('doppler1')
92 # data3= np.array(data3)
93 # data4 = np.array(data4)
94 # datacoh=datastuff.get('coherence2')
95 #
96 # datacohphase=datacoh*np.exp(-dataphase*1j)
97 # # data31 = np.fliplr(data3)
98 # # data41 = np.fliplr(data4)
99 #
100 # data31 = data3.reshape((1,data3.shape[0],data3.shape[1]))
101 # data41 = data4.reshape((1,data4.shape[0],data4.shape[1]))
102 # datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1]))
103 #
104 # datastack = np.vstack((data31,data41))
105 #
106 # self.dataOut.pairsList=[(0,1)]
107 # self.dataOut.data_cspc=datacohphase1.copy()
108 # self.dataOut.data_spc = datastack.copy()
109 # self.dataOut.channelList = range(2)
110 # self.dataOut.nProfiles = 25 #this!
111 # self.dataOut.nIncohInt = 1
112 # self.dataOut.nCohInt = 1 #this!
113 # self.dataOut.ippSeconds = 0.004 #this!
114 # self.dataOut.nFFTPoints = 25
115 # self.dataOut.utctime = self.utcfirst
116 # self.dataOut.heightList = np.array(datastuff.get('hts'))
117 #
118 # self.dataOut.flagNoData = False
119 #
120 # self.firsttime=False
121 #
122 # self.index+=1
123 #
124 #
125 # return foldercountercrosscheck, self.firsttime
126
127 def run(self,path=None,startDate=None, endDate=None,startTime=datetime.time(0,0,0),
128 endTime=datetime.time(23,59,59),walk=True,timezone='ut',
129 all=0,online=True,ext=None, **kwargs):
130
131 self.path=path
132 self.ext=ext
133 self.startDate=startDate
134 self.endDate=endDate
135 self.startTime=startTime
136 self.endTime=endTime
137 self.index=0
138
139 filelist = [ f for f in listdir(path)]
140 secondlist=[]
141 thirdlist=[]
142 utclist=[]
143 foldercountercheck=len([name for name in os.listdir(self.path)])
144
145 # ______________________________________________________________________________________________________________________________________________________________-
146 # First time through this acts much like the offline program,
147 # It reads in the latest file in the path to create a list. So, it finds
148 # the very last folder in the path, opens it and reads in the 30 matlab files
149 # inside of that. Then, this loop never runs again.
150
151
152 if (self.firsttime==True):
153 # self.FIRSTTIMERUNTHROUGH(path, self.startDate, self.endDate, self.startTime,
154 # self.endTime, walk, timezone, all, online, ext,filelist)
155 foldercountercrosscheck=0
156 self.utclist=utclist
157
158
30
159 for g in range(len(filelist)):
31 self.dataOut.pairsList=[(0,1)]
160 strsplit=filelist[g].split('.')
32 self.dataOut.channelList = range(2)
161 timeints=[int(i) for i in strsplit]
33 self.dataOut.nProfiles = 25 #this!
162 timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
34 self.dataOut.nIncohInt = 1
163 utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
35 self.dataOut.nCohInt = 1 #this!
164 secondlist.append(filelist[g])
36 self.dataOut.ippSeconds = 0.004 #this!
165 self.utclist.append(utctime)
37 self.dataOut.nFFTPoints = 25
166
38 self.dataOut.timeZone = 0
167 for k in range(len(secondlist)):
39 self.dataOut.heightList = np.array(datastuff.get('hts')).flatten()
168
169 path1=os.path.join(self.path,secondlist[k])
170 filecounter=len([name for name in os.listdir(path1)])
171 # print "Reading from this dir:" +path1
172 for r in range(filecounter):
173 matname=str(r)+'.mat'
174 bork=os.path.join(path1,matname)
175 # thirdlist is what ends up being the list to all matlab files
176 thirdlist.append(os.path.join(path1,matname))
177
178 # Set the index to -31 initially such that it reads the final 30 matlab files in the
179 # path for the first iteration. It gets reset in the continually checking portion.
180
40
181 self.utcfirst=utclist[-1]
41 def __readFile(self, currentfile):
182 self.list=thirdlist
42 print "Reading from this file:" + currentfile
183
43
184 currentfilenewest=self.list[self.indexfirsttime]
44 #filesplit=currentfile.split("\\")
185 print "Reading from this file:" + currentfilenewest
45 filesplit=currentfile.split("/")
186 filesplit=currentfilenewest.split("\\")
187 newsplit=filesplit[-2]
46 newsplit=filesplit[-2]
188 newnewsplit=newsplit.split(".")
47 newnewsplit=newsplit.split(".")
189 newnewsplit=[int(i) for i in newnewsplit]
48 newnewsplit=[int(i) for i in newnewsplit]
@@ -197,9 +56,20 class MatReader(ProcessingUnit):
197 goobnum=int(goobnum)
56 goobnum=int(goobnum)
198
57
199 self.utcfirst=self.utcfirst+goobnum*2
58 self.utcfirst=self.utcfirst+goobnum*2
59 # if (currentfile[43:]=='0.mat'):
60 # self.utcmatcounter=0
61 # self.utcfirst=self.utclist[self.index]
62
63 # if (self.utcmatcounter>60):
64 # self.utcmatcounter=0
65
66 # print self.utcmatcounter
200 print self.utcfirst
67 print self.utcfirst
68 try:
69 datastuff=sio.loadmat(currentfile)
70 except:
71 return None, None
201
72
202 datastuff=sio.loadmat(currentfilenewest)
203 dataphase=datastuff.get('phase')
73 dataphase=datastuff.get('phase')
204 data3=datastuff.get('doppler0')
74 data3=datastuff.get('doppler0')
205 data4=datastuff.get('doppler1')
75 data4=datastuff.get('doppler1')
@@ -217,121 +87,179 class MatReader(ProcessingUnit):
217
87
218 datastack = np.vstack((data31,data41))
88 datastack = np.vstack((data31,data41))
219
89
220 self.dataOut.pairsList=[(0,1)]
90 self.__setHeader(datastuff)
221 self.dataOut.data_cspc=datacohphase1.copy()
222 self.dataOut.data_spc = datastack.copy()
223 self.dataOut.channelList = range(2)
224 self.dataOut.nProfiles = 25 #this!
225 self.dataOut.nIncohInt = 1
226 self.dataOut.nCohInt = 1 #this!
227 self.dataOut.ippSeconds = 0.004 #this!
228 self.dataOut.nFFTPoints = 25
229 self.dataOut.utctime = self.utcfirst
230 self.dataOut.heightList = np.array(datastuff.get('hts'))
231
91
232 self.dataOut.flagNoData = False
92 spc = datastack
233 self.indexfirsttime+=1
93 cspc = datacohphase1
234
235 if (self.indexfirsttime>=30):
236 self.firsttime=False
237 self.index=0
238
94
95 return spc, cspc
239
96
240 #
97 def __findFiles(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
241
98
99 if startDate == None:
100 startDate = datetime.date(1970,1,1)
242
101
102 if endDate == None:
103 endDate = datetime.date(2050,1,1)
243
104
244 # __________________________________________________________________________________________________________________________________________________________-
105 startsearch1=datetime.datetime.combine(startDate,startTime)
106 startsearch2=(startsearch1-datetime.datetime(1970,1,1)).total_seconds()
107 endsearch1=datetime.datetime.combine(endDate,endTime)
108 endsearch2=(endsearch1-datetime.datetime(1970,1,1)).total_seconds()
245
109
110 dirList = listdir(path)
111 dirList = sorted(dirList)
246
112
247 # Now we check for new folders, if some are detected, we repeat the process
113 dirListFiltered=[]
248 self.firsttime=False
114 fileListFiltered=[]
115 utclist=[]
249
116
250 if(foldercountercheck>foldercountercrosscheck):
117 if not dirList:
251 foldercountercrosscheck=foldercountercheck
118 print "No directories found"
119 return []
252
120
121 for thisDir in dirList:
122 if not os.path.isdir(os.path.join(path, thisDir)):
123 continue
253
124
254 for g in range(len(filelist)):
125 strsplit=thisDir.split('.')
255 strsplit=filelist[g].split('.')
256 timeints=[int(i) for i in strsplit]
126 timeints=[int(i) for i in strsplit]
257 timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
127 timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
258 utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
128 utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
259 secondlist.append(filelist[g])
260 self.utclist.append(utctime)
261
129
262 for k in range(len(secondlist)):
130 if not self.online:
131 if (utctime > endsearch2):
132 continue
263
133
264 path1=os.path.join(self.path,secondlist[k])
134 if (utctime < startsearch2):
265 filecounter=len([name for name in os.listdir(path1)])
135 continue
266 # print "Reading from this dir:" +path1
267 for r in range(filecounter):
268 matname=str(r)+'.mat'
269 bork=os.path.join(path1,matname)
270 # thirdlist is what ends up being the list to all matlab files
271 thirdlist.append(os.path.join(path1,matname))
272
136
273 # Set the index to -31 initially such that it reads the final 30 matlab files in the
137 dirListFiltered.append(thisDir)
274 # path for the first iteration. It gets reset in the continually checking portion.
138 utclist.append(utctime)
275
139
276 self.utcfirst=utclist[-1]
140 if not dirListFiltered:
277 self.list=thirdlist
141 print "filtro"
142 return []
278
143
279 currentfilenewest=self.list[self.indexfirsttime]
144 for thisDir in dirListFiltered:
280 print "Reading from this file:" + currentfilenewest
281 filesplit=currentfilenewest.split("\\")
282 newsplit=filesplit[-2]
283 newnewsplit=newsplit.split(".")
284 newnewsplit=[int(i) for i in newnewsplit]
285 gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5])
286 self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds()
287
145
146 pathFile = os.path.join(self.path, thisDir)
288
147
289 newsplit=filesplit[-1]
148 fileList = os.listdir(pathFile)
290 newnewsplit=newsplit.split(".")
291 goobnum=newnewsplit[0]
292 goobnum=int(goobnum)
293
149
294 self.utcfirst=self.utcfirst+goobnum*2
150 if not fileList:
295 print "The utc of the file is:" + self.utcfirst
151 continue
296
152
297 datastuff=sio.loadmat(currentfilenewest)
153 for k in range(len(fileList)):
298 dataphase=datastuff.get('phase')
154 thisFile = str(k)+'.mat'
299 data3=datastuff.get('doppler0')
300 data4=datastuff.get('doppler1')
301 data3= np.array(data3)
302 data4 = np.array(data4)
303 datacoh=datastuff.get('coherence2')
304
155
305 datacohphase=datacoh*np.exp(-dataphase*1j)
156 if not os.path.isfile(os.path.join(pathFile, thisFile)):
306 # data31 = np.fliplr(data3)
157 continue
307 # data41 = np.fliplr(data4)
308
158
309 data31 = data3.reshape((1,data3.shape[0],data3.shape[1]))
159 fileListFiltered.append(os.path.join(pathFile, thisFile))
310 data41 = data4.reshape((1,data4.shape[0],data4.shape[1]))
311 datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1]))
312
160
313 datastack = np.vstack((data31,data41))
161 return fileListFiltered
314
162
315 self.dataOut.pairsList=[(0,1)]
163 def __getNextOnlineFile(self, seconds = 30):
316 self.dataOut.data_cspc=datacohphase1.copy()
317 self.dataOut.data_spc = datastack.copy()
318 self.dataOut.channelList = range(2)
319 self.dataOut.nProfiles = 25 #this!
320 self.dataOut.nIncohInt = 1
321 self.dataOut.nCohInt = 1 #this!
322 self.dataOut.ippSeconds = 0.004 #this!
323 self.dataOut.nFFTPoints = 25
324 self.dataOut.utctime = self.utcfirst
325 self.dataOut.heightList = np.array(datastuff.get('hts'))
326
164
327 self.dataOut.flagNoData = False
165 filename = self.__getNextOfflineFile()
328 self.indexfirsttime+=1
329
166
330 if (self.indexfirsttime>=30):
167 if filename:
331 self.firsttime=False
168 return filename
332 self.index=0
169
170 ncurrentfiles = len(self.fileList)
171
172 nTries = 0
173 while (True):
174 filelist = self.__findFiles(self.path)
175 if len(filelist) > ncurrentfiles:
176 break
177
178 nTries += 1
179
180 if nTries > 3:
181 break
182
183 print "Waiting %d seconds ..." %seconds
184 time.sleep(30)
185
186 if not (len(filelist) > ncurrentfiles):
187 return None
188
189 self.fileList = filelist
190 filename = self.__getNextOfflineFile()
191
192 return filename
193
194 def __getNextOfflineFile(self):
195
196 if self.index >= len(self.fileList):
197 return None
198
199 filename=self.fileList[self.index]
200 self.index += 1
201 return filename
202
203 def __getNextFile(self):
333
204
205 if self.online:
206 filename = self.__getNextOnlineFile()
334 else:
207 else:
335 time.sleep(15)
208 filename = self.__getNextOfflineFile()
209 return filename
210
211 def setup(self, path, startDate=None, endDate=None,startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59)):
212
213 fileList = self.__findFiles(path, startDate, endDate, startTime, endTime)
214
215 if self.online:
216 self.index = len(fileList) -1
217 else:
218 self.index = 0
219
220 self.fileList = fileList
221
222 print "fin setup"
223
224 def run(self,path=None,startDate=None, endDate=None,
225 startTime=datetime.time(0,0,0),
226 endTime=datetime.time(23,59,59),
227 walk=True,timezone='ut',
228 all=0,online=False,ext=None,**kwargs):
229
230 self.path=path
231 self.ext=ext
232 self.startDate=startDate
233 self.endDate=endDate
234 self.startTime=startTime
235 self.endTime=endTime
236 self.online = online
237 self.dataOut.flagNoData = True
238
239 if (self.firsttime==True):
240 self.setup(path, startDate, endDate, startTime, endTime)
241 self.firsttime=False
242
243
244 if not self.fileList:
245 self.dataOut.flagNoData = True
246 print "lista vacia"
247 return
248
249 currentfile = self.__getNextFile()
250
251 if not currentfile:
252 "no file"
253 return
254
255 spc, cspc = self.__readFile(currentfile)
256
257 if spc!=None:
258
259 self.dataOut.data_spc = spc
260 self.dataOut.data_cspc = cspc
261 self.dataOut.utctime = self.utcfirst
262 self.dataOut.flagNoData = False
336
263
337 return 1
264 return 1
265 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now