##// END OF EJS Templates
schain
RSS
CIRI_online_version
Alexander Valdez -
r591:99808b528056
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,337
1 from schainpy.model.data.jrodata import *
2 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
3 from schainpy.model.io.jroIO_base import *
4
5 import scipy.io as sio
6 import pprint
7 import numpy as np
8 from os import listdir
9 from os.path import isfile, join
10 import datetime
11 import cmath
12 from astropy.io.ascii.tests.test_connect import files
13
14 # Path needs to be whereever the matlab data files are stored.
15 # This program works by reading how many data folders remain inside of the
16 # path destination. Every iteration of the program it will re-search
17 # the path destination, if new folder(s) are detected, it will read
18 # in all data from say... enough files to cover an hour and generate
19 # spectra and RTI plots.
20
21 class MatReader(ProcessingUnit):
22
23 index=None
24 list=None
25 firsttime=True
26 utccounter=None
27 utcfiletime=None
28 utcmatcounter=0
29 utcfirst=None
30 utclist=None
31 foldercountercrosscheck=None
32 foldercountercheck=None
33 indexfirsttime=-31
34
35 def __init__(self):
36 self.dataOut = Spectra()
37 return
38
39 # def FIRSTTIMERUNTHROUGH(self,path=None,startDate=None, endDate=None,startTime=datetime.time(0,0,0),
40 # endTime=datetime.time(23,59,59),walk=True,timezone='ut',
41 # all=0,online=True,ext=None,filelist=None, **kwargs):
42 #
43 # foldcountercrosscheck=0
44 # self.foldercountercrosscheck=foldcountercrosscheck
45 #
46 # for g in range(len(filelist)):
47 # strsplit=filelist[g].split('.')
48 # timeints=[int(i) for i in strsplit]
49 # timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
50 # utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
51 # secondlist.append(filelist[g])
52 # self.utclist.append(utctime)
53 #
54 # for k in range(len(secondlist)):
55 #
56 # path1=os.path.join(self.path,secondlist[k])
57 # filecounter=len([name for name in os.listdir(path1)])
58 # # print "Reading from this dir:" +path1
59 # for r in range(filecounter):
60 # matname=str(r)+'.mat'
61 # bork=os.path.join(path1,matname)
62 # # thirdlist is what ends up being the list to all matlab files
63 # thirdlist.append(os.path.join(path1,matname))
64 #
65 # self.utcfirst=utclist[-1]
66 # self.index=-31
67 # self.list=thirdlist
68 #
69 # while (self.index <= -1):
70 # currentfilenewest=self.list[self.index]
71 # print "Reading from this file:" + currentfilenewest
72 # filesplit=currentfilenewest.split("\\")
73 # newsplit=filesplit[-2]
74 # newnewsplit=newsplit.split(".")
75 # newnewsplit=[int(i) for i in newnewsplit]
76 # gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5])
77 # self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds()
78 #
79 #
80 # newsplit=filesplit[-1]
81 # newnewsplit=newsplit.split(".")
82 # goobnum=newnewsplit[0]
83 # goobnum=int(goobnum)
84 #
85 # self.utcfirst=self.utcfirst+goobnum*2
86 # print self.utcfirst
87 #
88 # datastuff=sio.loadmat(currentfilenewest)
89 # dataphase=datastuff.get('phase')
90 # data3=datastuff.get('doppler0')
91 # data4=datastuff.get('doppler1')
92 # data3= np.array(data3)
93 # data4 = np.array(data4)
94 # datacoh=datastuff.get('coherence2')
95 #
96 # datacohphase=datacoh*np.exp(-dataphase*1j)
97 # # data31 = np.fliplr(data3)
98 # # data41 = np.fliplr(data4)
99 #
100 # data31 = data3.reshape((1,data3.shape[0],data3.shape[1]))
101 # data41 = data4.reshape((1,data4.shape[0],data4.shape[1]))
102 # datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1]))
103 #
104 # datastack = np.vstack((data31,data41))
105 #
106 # self.dataOut.pairsList=[(0,1)]
107 # self.dataOut.data_cspc=datacohphase1.copy()
108 # self.dataOut.data_spc = datastack.copy()
109 # self.dataOut.channelList = range(2)
110 # self.dataOut.nProfiles = 25 #this!
111 # self.dataOut.nIncohInt = 1
112 # self.dataOut.nCohInt = 1 #this!
113 # self.dataOut.ippSeconds = 0.004 #this!
114 # self.dataOut.nFFTPoints = 25
115 # self.dataOut.utctime = self.utcfirst
116 # self.dataOut.heightList = np.array(datastuff.get('hts'))
117 #
118 # self.dataOut.flagNoData = False
119 #
120 # self.firsttime=False
121 #
122 # self.index+=1
123 #
124 #
125 # return foldercountercrosscheck, self.firsttime
126
127 def run(self,path=None,startDate=None, endDate=None,startTime=datetime.time(0,0,0),
128 endTime=datetime.time(23,59,59),walk=True,timezone='ut',
129 all=0,online=True,ext=None, **kwargs):
130
131 self.path=path
132 self.ext=ext
133 self.startDate=startDate
134 self.endDate=endDate
135 self.startTime=startTime
136 self.endTime=endTime
137 self.index=0
138
139 filelist = [ f for f in listdir(path)]
140 secondlist=[]
141 thirdlist=[]
142 utclist=[]
143 foldercountercheck=len([name for name in os.listdir(self.path)])
144
145 # ______________________________________________________________________________________________________________________________________________________________-
146 # First time through this acts much like the offline program,
147 # It reads in the latest file in the path to create a list. So, it finds
148 # the very last folder in the path, opens it and reads in the 30 matlab files
149 # inside of that. Then, this loop never runs again.
150
151
152 if (self.firsttime==True):
153 # self.FIRSTTIMERUNTHROUGH(path, self.startDate, self.endDate, self.startTime,
154 # self.endTime, walk, timezone, all, online, ext,filelist)
155 foldercountercrosscheck=0
156 self.utclist=utclist
157
158
159 for g in range(len(filelist)):
160 strsplit=filelist[g].split('.')
161 timeints=[int(i) for i in strsplit]
162 timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
163 utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
164 secondlist.append(filelist[g])
165 self.utclist.append(utctime)
166
167 for k in range(len(secondlist)):
168
169 path1=os.path.join(self.path,secondlist[k])
170 filecounter=len([name for name in os.listdir(path1)])
171 # print "Reading from this dir:" +path1
172 for r in range(filecounter):
173 matname=str(r)+'.mat'
174 bork=os.path.join(path1,matname)
175 # thirdlist is what ends up being the list to all matlab files
176 thirdlist.append(os.path.join(path1,matname))
177
178 # Set the index to -31 initially such that it reads the final 30 matlab files in the
179 # path for the first iteration. It gets reset in the continually checking portion.
180
181 self.utcfirst=utclist[-1]
182 self.list=thirdlist
183
184 currentfilenewest=self.list[self.indexfirsttime]
185 print "Reading from this file:" + currentfilenewest
186 filesplit=currentfilenewest.split("\\")
187 newsplit=filesplit[-2]
188 newnewsplit=newsplit.split(".")
189 newnewsplit=[int(i) for i in newnewsplit]
190 gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5])
191 self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds()
192
193
194 newsplit=filesplit[-1]
195 newnewsplit=newsplit.split(".")
196 goobnum=newnewsplit[0]
197 goobnum=int(goobnum)
198
199 self.utcfirst=self.utcfirst+goobnum*2
200 print self.utcfirst
201
202 datastuff=sio.loadmat(currentfilenewest)
203 dataphase=datastuff.get('phase')
204 data3=datastuff.get('doppler0')
205 data4=datastuff.get('doppler1')
206 data3= np.array(data3)
207 data4 = np.array(data4)
208 datacoh=datastuff.get('coherence2')
209
210 datacohphase=datacoh*np.exp(-dataphase*1j)
211 # data31 = np.fliplr(data3)
212 # data41 = np.fliplr(data4)
213
214 data31 = data3.reshape((1,data3.shape[0],data3.shape[1]))
215 data41 = data4.reshape((1,data4.shape[0],data4.shape[1]))
216 datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1]))
217
218 datastack = np.vstack((data31,data41))
219
220 self.dataOut.pairsList=[(0,1)]
221 self.dataOut.data_cspc=datacohphase1.copy()
222 self.dataOut.data_spc = datastack.copy()
223 self.dataOut.channelList = range(2)
224 self.dataOut.nProfiles = 25 #this!
225 self.dataOut.nIncohInt = 1
226 self.dataOut.nCohInt = 1 #this!
227 self.dataOut.ippSeconds = 0.004 #this!
228 self.dataOut.nFFTPoints = 25
229 self.dataOut.utctime = self.utcfirst
230 self.dataOut.heightList = np.array(datastuff.get('hts'))
231
232 self.dataOut.flagNoData = False
233 self.indexfirsttime+=1
234
235 if (self.indexfirsttime>=30):
236 self.firsttime=False
237 self.index=0
238
239
240 #
241
242
243
244 # __________________________________________________________________________________________________________________________________________________________-
245
246
247 # Now we check for new folders, if some are detected, we repeat the process
248 self.firsttime=False
249
250 if(foldercountercheck>foldercountercrosscheck):
251 foldercountercrosscheck=foldercountercheck
252
253
254 for g in range(len(filelist)):
255 strsplit=filelist[g].split('.')
256 timeints=[int(i) for i in strsplit]
257 timelist=datetime.datetime(timeints[0],timeints[1],timeints[2],timeints[3],timeints[4],timeints[5])
258 utctime=(timelist-datetime.datetime(1970,1,1)).total_seconds()
259 secondlist.append(filelist[g])
260 self.utclist.append(utctime)
261
262 for k in range(len(secondlist)):
263
264 path1=os.path.join(self.path,secondlist[k])
265 filecounter=len([name for name in os.listdir(path1)])
266 # print "Reading from this dir:" +path1
267 for r in range(filecounter):
268 matname=str(r)+'.mat'
269 bork=os.path.join(path1,matname)
270 # thirdlist is what ends up being the list to all matlab files
271 thirdlist.append(os.path.join(path1,matname))
272
273 # Set the index to -31 initially such that it reads the final 30 matlab files in the
274 # path for the first iteration. It gets reset in the continually checking portion.
275
276 self.utcfirst=utclist[-1]
277 self.list=thirdlist
278
279 currentfilenewest=self.list[self.indexfirsttime]
280 print "Reading from this file:" + currentfilenewest
281 filesplit=currentfilenewest.split("\\")
282 newsplit=filesplit[-2]
283 newnewsplit=newsplit.split(".")
284 newnewsplit=[int(i) for i in newnewsplit]
285 gooblist=datetime.datetime(newnewsplit[0],newnewsplit[1],newnewsplit[2],newnewsplit[3],newnewsplit[4],newnewsplit[5])
286 self.utcfirst=(gooblist-datetime.datetime(1970,1,1)).total_seconds()
287
288
289 newsplit=filesplit[-1]
290 newnewsplit=newsplit.split(".")
291 goobnum=newnewsplit[0]
292 goobnum=int(goobnum)
293
294 self.utcfirst=self.utcfirst+goobnum*2
295 print "The utc of the file is:" + self.utcfirst
296
297 datastuff=sio.loadmat(currentfilenewest)
298 dataphase=datastuff.get('phase')
299 data3=datastuff.get('doppler0')
300 data4=datastuff.get('doppler1')
301 data3= np.array(data3)
302 data4 = np.array(data4)
303 datacoh=datastuff.get('coherence2')
304
305 datacohphase=datacoh*np.exp(-dataphase*1j)
306 # data31 = np.fliplr(data3)
307 # data41 = np.fliplr(data4)
308
309 data31 = data3.reshape((1,data3.shape[0],data3.shape[1]))
310 data41 = data4.reshape((1,data4.shape[0],data4.shape[1]))
311 datacohphase1 = datacohphase.reshape((1,datacoh.shape[0],datacoh.shape[1]))
312
313 datastack = np.vstack((data31,data41))
314
315 self.dataOut.pairsList=[(0,1)]
316 self.dataOut.data_cspc=datacohphase1.copy()
317 self.dataOut.data_spc = datastack.copy()
318 self.dataOut.channelList = range(2)
319 self.dataOut.nProfiles = 25 #this!
320 self.dataOut.nIncohInt = 1
321 self.dataOut.nCohInt = 1 #this!
322 self.dataOut.ippSeconds = 0.004 #this!
323 self.dataOut.nFFTPoints = 25
324 self.dataOut.utctime = self.utcfirst
325 self.dataOut.heightList = np.array(datastuff.get('hts'))
326
327 self.dataOut.flagNoData = False
328 self.indexfirsttime+=1
329
330 if (self.indexfirsttime>=30):
331 self.firsttime=False
332 self.index=0
333
334 else:
335 time.sleep(15)
336
337 return 1 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now