##// END OF EJS Templates
schain
RSS
Changes for Online Reading
Daniel Valdez -
r508:718a5f058f7f
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,619 +1,678
1 '''
1 '''
2 @author: Daniel Suarez
2 @author: Daniel Suarez
3 '''
3 '''
4
4
5 import os
5 import os
6 import sys
6 import sys
7 import glob
7 import glob
8 import fnmatch
8 import fnmatch
9 import datetime
9 import datetime
10 import time
10 import time
11 import re
11 import re
12 import h5py
12 import h5py
13 import numpy
13 import numpy
14
14
15 from model.proc.jroproc_base import ProcessingUnit, Operation
15 from model.proc.jroproc_base import ProcessingUnit, Operation
16 from model.data.jroamisr import AMISR
16 from model.data.jroamisr import AMISR
17
17
18 class RadacHeader():
18 class RadacHeader():
19 def __init__(self, fp):
19 def __init__(self, fp):
20 header = 'Raw11/Data/RadacHeader'
20 header = 'Raw11/Data/RadacHeader'
21 self.beamCodeByPulse = fp.get(header+'/BeamCode')
21 self.beamCodeByPulse = fp.get(header+'/BeamCode')
22 self.beamCode = fp.get('Raw11/Data/Beamcodes')
22 self.beamCode = fp.get('Raw11/Data/Beamcodes')
23 self.code = fp.get(header+'/Code')
23 self.code = fp.get(header+'/Code')
24 self.frameCount = fp.get(header+'/FrameCount')
24 self.frameCount = fp.get(header+'/FrameCount')
25 self.modeGroup = fp.get(header+'/ModeGroup')
25 self.modeGroup = fp.get(header+'/ModeGroup')
26 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')
26 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')
27 self.pulseCount = fp.get(header+'/PulseCount')
27 self.pulseCount = fp.get(header+'/PulseCount')
28 self.radacTime = fp.get(header+'/RadacTime')
28 self.radacTime = fp.get(header+'/RadacTime')
29 self.timeCount = fp.get(header+'/TimeCount')
29 self.timeCount = fp.get(header+'/TimeCount')
30 self.timeStatus = fp.get(header+'/TimeStatus')
30 self.timeStatus = fp.get(header+'/TimeStatus')
31
31
32 self.nrecords = self.pulseCount.shape[0] #nblocks
32 self.nrecords = self.pulseCount.shape[0] #nblocks
33 self.npulses = self.pulseCount.shape[1] #nprofile
33 self.npulses = self.pulseCount.shape[1] #nprofile
34 self.nsamples = self.nsamplesPulse[0,0] #ngates
34 self.nsamples = self.nsamplesPulse[0,0] #ngates
35 self.nbeams = self.beamCode.shape[1]
35 self.nbeams = self.beamCode.shape[1]
36
36
37
37
38 def getIndexRangeToPulse(self, idrecord=0):
38 def getIndexRangeToPulse(self, idrecord=0):
39 #indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
39 #indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
40 #startPulseCountId = indexToZero[0][0]
40 #startPulseCountId = indexToZero[0][0]
41 #endPulseCountId = startPulseCountId - 1
41 #endPulseCountId = startPulseCountId - 1
42 #range1 = numpy.arange(startPulseCountId,self.npulses,1)
42 #range1 = numpy.arange(startPulseCountId,self.npulses,1)
43 #range2 = numpy.arange(0,startPulseCountId,1)
43 #range2 = numpy.arange(0,startPulseCountId,1)
44 #return range1, range2
44 #return range1, range2
45
45 zero = 0
46 looking_zeros_index = numpy.where(self.pulseCount.value[idrecord,:]==0)[0]
46 npulse = max(self.pulseCount[0,:]+1)-1
47 getLastIndexZero = looking_zeros_index[-1]
47 looking_index = numpy.where(self.pulseCount.value[idrecord,:]==npulse)[0]
48 index_data = numpy.arange(0,getLastIndexZero,1)
48 getLastIndex = looking_index[-1]
49 index_buffer = numpy.arange(getLastIndexZero,self.npulses,1)
49 index_data = numpy.arange(0,getLastIndex+1,1)
50 index_buffer = numpy.arange(getLastIndex+1,self.npulses,1)
50 return index_data, index_buffer
51 return index_data, index_buffer
51
52
52 class AMISRReader(ProcessingUnit):
53 class AMISRReader(ProcessingUnit):
53
54
54 path = None
55 path = None
55 startDate = None
56 startDate = None
56 endDate = None
57 endDate = None
57 startTime = None
58 startTime = None
58 endTime = None
59 endTime = None
59 walk = None
60 walk = None
60 isConfig = False
61 isConfig = False
61
62
62 def __init__(self):
63 def __init__(self):
63 self.set = None
64 self.set = None
64 self.subset = None
65 self.subset = None
65 self.extension_file = '.h5'
66 self.extension_file = '.h5'
66 self.dtc_str = 'dtc'
67 self.dtc_str = 'dtc'
67 self.dtc_id = 0
68 self.dtc_id = 0
68 self.status = True
69 self.status = True
69 self.isConfig = False
70 self.isConfig = False
70 self.dirnameList = []
71 self.dirnameList = []
71 self.filenameList = []
72 self.filenameList = []
72 self.fileIndex = None
73 self.fileIndex = None
73 self.flagNoMoreFiles = False
74 self.flagNoMoreFiles = False
74 self.flagIsNewFile = 0
75 self.flagIsNewFile = 0
75 self.filename = ''
76 self.filename = ''
76 self.amisrFilePointer = None
77 self.amisrFilePointer = None
77 self.radacHeaderObj = None
78 self.radacHeaderObj = None
78 self.dataOut = self.__createObjByDefault()
79 self.dataOut = self.__createObjByDefault()
79 self.datablock = None
80 self.datablock = None
80 self.rest_datablock = None
81 self.rest_datablock = None
81 self.range = None
82 self.range = None
82 self.idrecord_count = 0
83 self.idrecord_count = 0
83 self.profileIndex = 0
84 self.profileIndex = 0
84 self.index_amisr_sample = None
85 self.index_amisr_sample = None
85 self.index_amisr_buffer = None
86 self.index_amisr_buffer = None
86 self.beamCodeByFrame = None
87 self.beamCodeByFrame = None
87 self.radacTimeByFrame = None
88 self.radacTimeByFrame = None
88 #atributos originales tal y como esta en el archivo de datos
89 #atributos originales tal y como esta en el archivo de datos
89 self.beamCodesFromFile = None
90 self.beamCodesFromFile = None
90 self.radacTimeFromFile = None
91 self.radacTimeFromFile = None
91 self.rangeFromFile = None
92 self.rangeFromFile = None
92 self.dataByFrame = None
93 self.dataByFrame = None
93 self.dataset = None
94 self.dataset = None
94
95
95 self.beamCodeDict = {}
96 self.beamCodeDict = {}
96 self.beamRangeDict = {}
97 self.beamRangeDict = {}
97
98
98 #experiment cgf file
99 #experiment cgf file
99 self.npulsesint_fromfile = None
100 self.npulsesint_fromfile = None
100 self.recordsperfile_fromfile = None
101 self.recordsperfile_fromfile = None
101 self.nbeamcodes_fromfile = None
102 self.nbeamcodes_fromfile = None
102 self.ngates_fromfile = None
103 self.ngates_fromfile = None
103 self.ippSeconds_fromfile = None
104 self.ippSeconds_fromfile = None
104 self.frequency_h5file = None
105 self.frequency_h5file = None
105
106
106
107
107 self.__firstFile = True
108 self.__firstFile = True
108 self.buffer_radactime = None
109 self.buffer_radactime = None
109
110
110 self.index4_schain_datablock = None
111 self.index4_schain_datablock = None
111 self.index4_buffer = None
112 self.index4_buffer = None
112 self.schain_datablock = None
113 self.schain_datablock = None
113 self.buffer = None
114 self.buffer = None
114 self.linear_pulseCount = None
115 self.linear_pulseCount = None
115 self.npulseByFrame = None
116 self.npulseByFrame = None
116 self.profileIndex_offset = None
117 self.profileIndex_offset = None
117 self.timezone = 'ut'
118 self.timezone = 'ut'
118
119
120 self.__waitForNewFile = 20
121 self.__filename_online = None
122
119 def __createObjByDefault(self):
123 def __createObjByDefault(self):
120
124
121 dataObj = AMISR()
125 dataObj = AMISR()
122
126
123 return dataObj
127 return dataObj
124
128
125 def __setParameters(self,path,startDate,endDate,startTime,endTime,walk):
129 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
126 self.path = path
130 self.path = path
127 self.startDate = startDate
131 self.startDate = startDate
128 self.endDate = endDate
132 self.endDate = endDate
129 self.startTime = startTime
133 self.startTime = startTime
130 self.endTime = endTime
134 self.endTime = endTime
131 self.walk = walk
135 self.walk = walk
132
136
133 def __checkPath(self):
137 def __checkPath(self):
134 if os.path.exists(self.path):
138 if os.path.exists(self.path):
135 self.status = 1
139 self.status = 1
136 else:
140 else:
137 self.status = 0
141 self.status = 0
138 print 'Path:%s does not exists'%self.path
142 print 'Path:%s does not exists'%self.path
139
143
140 return
144 return
141
145
142 def __selDates(self, amisr_dirname_format):
146 def __selDates(self, amisr_dirname_format):
143 try:
147 try:
144 year = int(amisr_dirname_format[0:4])
148 year = int(amisr_dirname_format[0:4])
145 month = int(amisr_dirname_format[4:6])
149 month = int(amisr_dirname_format[4:6])
146 dom = int(amisr_dirname_format[6:8])
150 dom = int(amisr_dirname_format[6:8])
147 thisDate = datetime.date(year,month,dom)
151 thisDate = datetime.date(year,month,dom)
148
152
149 if (thisDate>=self.startDate and thisDate <= self.endDate):
153 if (thisDate>=self.startDate and thisDate <= self.endDate):
150 return amisr_dirname_format
154 return amisr_dirname_format
151 except:
155 except:
152 return None
156 return None
153
157
154 def __findDataForDates(self):
158 def __findDataForDates(self,online=False):
155
159
156
160
157
161
158 if not(self.status):
162 if not(self.status):
159 return None
163 return None
160
164
161 pat = '\d+.\d+'
165 pat = '\d+.\d+'
162 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
166 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
163 dirnameList = filter(lambda x:x!=None,dirnameList)
167 dirnameList = filter(lambda x:x!=None,dirnameList)
164 dirnameList = [x.string for x in dirnameList]
168 dirnameList = [x.string for x in dirnameList]
165 dirnameList = [self.__selDates(x) for x in dirnameList]
169 if not(online):
166 dirnameList = filter(lambda x:x!=None,dirnameList)
170 dirnameList = [self.__selDates(x) for x in dirnameList]
171 dirnameList = filter(lambda x:x!=None,dirnameList)
167 if len(dirnameList)>0:
172 if len(dirnameList)>0:
168 self.status = 1
173 self.status = 1
169 self.dirnameList = dirnameList
174 self.dirnameList = dirnameList
170 self.dirnameList.sort()
175 self.dirnameList.sort()
171 else:
176 else:
172 self.status = 0
177 self.status = 0
173 return None
178 return None
174
179
175 def __getTimeFromData(self):
180 def __getTimeFromData(self):
176 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
181 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
177 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
182 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
178
183
179 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
184 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
180 print '........................................'
185 print '........................................'
181 filter_filenameList = []
186 filter_filenameList = []
182 self.filenameList.sort()
187 self.filenameList.sort()
183 for i in range(len(self.filenameList)-1):
188 for i in range(len(self.filenameList)-1):
184 filename = self.filenameList[i]
189 filename = self.filenameList[i]
185 fp = h5py.File(filename,'r')
190 fp = h5py.File(filename,'r')
186 time_str = fp.get('Time/RadacTimeString')
191 time_str = fp.get('Time/RadacTimeString')
187
192
188 startDateTimeStr_File = time_str[0][0].split('.')[0]
193 startDateTimeStr_File = time_str[0][0].split('.')[0]
189 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
194 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
190 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
195 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
191
196
192 endDateTimeStr_File = time_str[-1][-1].split('.')[0]
197 endDateTimeStr_File = time_str[-1][-1].split('.')[0]
193 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
198 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
194 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
199 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
195
200
196 fp.close()
201 fp.close()
197
202
198 if self.timezone == 'lt':
203 if self.timezone == 'lt':
199 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
204 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
200 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
205 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
201
206
202 if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
207 if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
203 #self.filenameList.remove(filename)
208 #self.filenameList.remove(filename)
204 filter_filenameList.append(filename)
209 filter_filenameList.append(filename)
205
210
206 filter_filenameList.sort()
211 filter_filenameList.sort()
207 self.filenameList = filter_filenameList
212 self.filenameList = filter_filenameList
208 return 1
213 return 1
209
214
210 def __filterByGlob1(self, dirName):
215 def __filterByGlob1(self, dirName):
211 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
216 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
212 filterDict = {}
217 filterDict = {}
213 filterDict.setdefault(dirName)
218 filterDict.setdefault(dirName)
214 filterDict[dirName] = filter_files
219 filterDict[dirName] = filter_files
215 return filterDict
220 return filterDict
216
221
217 def __getFilenameList(self, fileListInKeys, dirList):
222 def __getFilenameList(self, fileListInKeys, dirList):
218 for value in fileListInKeys:
223 for value in fileListInKeys:
219 dirName = value.keys()[0]
224 dirName = value.keys()[0]
220 for file in value[dirName]:
225 for file in value[dirName]:
221 filename = os.path.join(dirName, file)
226 filename = os.path.join(dirName, file)
222 self.filenameList.append(filename)
227 self.filenameList.append(filename)
223
228
224
229
225 def __selectDataForTimes(self):
230 def __selectDataForTimes(self, online=False):
226 #aun no esta implementado el filtro for tiempo
231 #aun no esta implementado el filtro for tiempo
227 if not(self.status):
232 if not(self.status):
228 return None
233 return None
229
234
230 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
235 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
231
236
232 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
237 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
233
238
234 self.__getFilenameList(fileListInKeys, dirList)
239 self.__getFilenameList(fileListInKeys, dirList)
235 #filtro por tiempo
240 if not(online):
236 if not(self.all):
241 #filtro por tiempo
237 self.__getTimeFromData()
242 if not(self.all):
243 self.__getTimeFromData()
238
244
239
245 if len(self.filenameList)>0:
240 if len(self.filenameList)>0:
246 self.status = 1
241 self.status = 1
247 self.filenameList.sort()
242 self.filenameList.sort()
248 else:
249 self.status = 0
250 return None
251 return 1
243 else:
252 else:
244 self.status = 0
253 #get the last file - 1
245 return None
254 self.filenameList = [self.filenameList[-2]]
255 return 1
256
257 def __searchFilesOnline(self,
258 path,
259 walk=True):
260
261 startDate = datetime.datetime.utcnow().date()
262 endDate = datetime.datetime.utcnow().date()
263
264 self.__setParameters(path=path, startDate=startDate, endDate=endDate, walk=walk)
265
266 self.__checkPath()
267
268 self.__findDataForDates(online=True)
269
270 self.dirnameList = [self.dirnameList[-1]]
271
272 self.__selectDataForTimes(online=True)
273
274 return
246
275
247
276
248 def __searchFilesOffline(self,
277 def __searchFilesOffline(self,
249 path,
278 path,
250 startDate,
279 startDate,
251 endDate,
280 endDate,
252 startTime=datetime.time(0,0,0),
281 startTime=datetime.time(0,0,0),
253 endTime=datetime.time(23,59,59),
282 endTime=datetime.time(23,59,59),
254 walk=True):
283 walk=True):
255
284
256 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
285 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
257
286
258 self.__checkPath()
287 self.__checkPath()
259
288
260 self.__findDataForDates()
289 self.__findDataForDates()
261
290
262 self.__selectDataForTimes()
291 self.__selectDataForTimes()
263
292
264 for i in range(len(self.filenameList)):
293 for i in range(len(self.filenameList)):
265 print "%s" %(self.filenameList[i])
294 print "%s" %(self.filenameList[i])
266
295
267 return
296 return
268
297
269 def __setNextFileOffline(self):
298 def __setNextFileOffline(self):
270 idFile = self.fileIndex
299 idFile = self.fileIndex
271
300
272 while (True):
301 while (True):
273 idFile += 1
302 idFile += 1
274 if not(idFile < len(self.filenameList)):
303 if not(idFile < len(self.filenameList)):
275 self.flagNoMoreFiles = 1
304 self.flagNoMoreFiles = 1
276 print "No more Files"
305 print "No more Files"
277 return 0
306 return 0
278
307
279 filename = self.filenameList[idFile]
308 filename = self.filenameList[idFile]
280
309
281 amisrFilePointer = h5py.File(filename,'r')
310 amisrFilePointer = h5py.File(filename,'r')
282
311
283 break
312 break
284
313
285 self.flagIsNewFile = 1
314 self.flagIsNewFile = 1
286 self.fileIndex = idFile
315 self.fileIndex = idFile
287 self.filename = filename
316 self.filename = filename
288
317
289 self.amisrFilePointer = amisrFilePointer
318 self.amisrFilePointer = amisrFilePointer
290
319
291 print "Setting the file: %s"%self.filename
320 print "Setting the file: %s"%self.filename
292
321
293 return 1
322 return 1
294
323
324
325 def __setNextFileOnline(self):
326 filename = self.filenameList[0]
327 if self.__filename_online != None:
328 self.__selectDataForTimes(online=True)
329 filename = self.filenameList[0]
330 while self.__filename_online == filename:
331 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
332 time.sleep(self.__waitForNewFile)
333 self.__selectDataForTimes(online=True)
334 filename = self.filenameList[0]
335
336 self.__filename_online = filename
337
338 self.amisrFilePointer = h5py.File(filename,'r')
339 self.flagIsNewFile = 1
340 self.filename = filename
341 print "Setting the file: %s"%self.filename
342 return 1
343
344
295 def __readHeader(self):
345 def __readHeader(self):
296 self.radacHeaderObj = RadacHeader(self.amisrFilePointer)
346 self.radacHeaderObj = RadacHeader(self.amisrFilePointer)
297
347
298 #update values from experiment cfg file
348 #update values from experiment cfg file
299 if self.radacHeaderObj.nrecords == self.recordsperfile_fromfile:
349 if self.radacHeaderObj.nrecords == self.recordsperfile_fromfile:
300 self.radacHeaderObj.nrecords = self.recordsperfile_fromfile
350 self.radacHeaderObj.nrecords = self.recordsperfile_fromfile
301 self.radacHeaderObj.nbeams = self.nbeamcodes_fromfile
351 self.radacHeaderObj.nbeams = self.nbeamcodes_fromfile
302 self.radacHeaderObj.npulses = self.npulsesint_fromfile
352 self.radacHeaderObj.npulses = self.npulsesint_fromfile
303 self.radacHeaderObj.nsamples = self.ngates_fromfile
353 self.radacHeaderObj.nsamples = self.ngates_fromfile
304
354
305 #looking index list for data
355 #looking index list for data
306 start_index = self.radacHeaderObj.pulseCount[0,:][0]
356 start_index = self.radacHeaderObj.pulseCount[0,:][0]
307 end_index = self.radacHeaderObj.npulses
357 end_index = self.radacHeaderObj.npulses
308 range4data = range(start_index, end_index)
358 range4data = range(start_index, end_index)
309 self.index4_schain_datablock = numpy.array(range4data)
359 self.index4_schain_datablock = numpy.array(range4data)
310
360
311 buffer_start_index = 0
361 buffer_start_index = 0
312 buffer_end_index = self.radacHeaderObj.pulseCount[0,:][0]
362 buffer_end_index = self.radacHeaderObj.pulseCount[0,:][0]
313 range4buffer = range(buffer_start_index, buffer_end_index)
363 range4buffer = range(buffer_start_index, buffer_end_index)
314 self.index4_buffer = numpy.array(range4buffer)
364 self.index4_buffer = numpy.array(range4buffer)
315
365
316 self.linear_pulseCount = numpy.array(range4data + range4buffer)
366 self.linear_pulseCount = numpy.array(range4data + range4buffer)
317 self.npulseByFrame = max(self.radacHeaderObj.pulseCount[0,:]+1)
367 self.npulseByFrame = max(self.radacHeaderObj.pulseCount[0,:]+1)
318
368
319 #get tuning frequency
369 #get tuning frequency
320 frequency_h5file_dataset = self.amisrFilePointer.get('Rx'+'/TuningFrequency')
370 frequency_h5file_dataset = self.amisrFilePointer.get('Rx'+'/TuningFrequency')
321 self.frequency_h5file = frequency_h5file_dataset[0,0]
371 self.frequency_h5file = frequency_h5file_dataset[0,0]
322
372
323 self.flagIsNewFile = 1
373 self.flagIsNewFile = 1
324
374
325 def __getBeamCode(self):
375 def __getBeamCode(self):
326 self.beamCodeDict = {}
376 self.beamCodeDict = {}
327 self.beamRangeDict = {}
377 self.beamRangeDict = {}
328
378
329 beamCodeMap = self.amisrFilePointer.get('Setup/BeamcodeMap')
379 beamCodeMap = self.amisrFilePointer.get('Setup/BeamcodeMap')
330
380
331 for i in range(len(self.radacHeaderObj.beamCode[0,:])):
381 for i in range(len(self.radacHeaderObj.beamCode[0,:])):
332 self.beamCodeDict.setdefault(i)
382 self.beamCodeDict.setdefault(i)
333 self.beamRangeDict.setdefault(i)
383 self.beamRangeDict.setdefault(i)
334 beamcodeValue = self.radacHeaderObj.beamCode[0,i]
384 beamcodeValue = self.radacHeaderObj.beamCode[0,i]
335 beamcodeIndex = numpy.where(beamCodeMap[:,0] == beamcodeValue)[0][0]
385 beamcodeIndex = numpy.where(beamCodeMap[:,0] == beamcodeValue)[0][0]
336 x = beamCodeMap[beamcodeIndex][1]
386 x = beamCodeMap[beamcodeIndex][1]
337 y = beamCodeMap[beamcodeIndex][2]
387 y = beamCodeMap[beamcodeIndex][2]
338 z = beamCodeMap[beamcodeIndex][3]
388 z = beamCodeMap[beamcodeIndex][3]
339 self.beamCodeDict[i] = [beamcodeValue, x, y, z]
389 self.beamCodeDict[i] = [beamcodeValue, x, y, z]
340
390
341 just4record0 = self.radacHeaderObj.beamCodeByPulse[0,:]
391 just4record0 = self.radacHeaderObj.beamCodeByPulse[0,:]
342
392
343 for i in range(len(self.beamCodeDict.values())):
393 for i in range(len(self.beamCodeDict.values())):
344 xx = numpy.where(just4record0==self.beamCodeDict.values()[i][0])
394 xx = numpy.where(just4record0==self.beamCodeDict.values()[i][0])
345 indexPulseByBeam = self.linear_pulseCount[xx[0]]
395 indexPulseByBeam = self.linear_pulseCount[xx[0]]
346 self.beamRangeDict[i] = indexPulseByBeam
396 self.beamRangeDict[i] = indexPulseByBeam
347
397
348 def __getExpParameters(self):
398 def __getExpParameters(self):
349 if not(self.status):
399 if not(self.status):
350 return None
400 return None
351
401
352 experimentCfgPath = os.path.join(self.path, self.dirnameList[0], 'Setup')
402 experimentCfgPath = os.path.join(self.path, self.dirnameList[0], 'Setup')
353
403
354 expFinder = glob.glob1(experimentCfgPath,'*.exp')
404 expFinder = glob.glob1(experimentCfgPath,'*.exp')
355 if len(expFinder)== 0:
405 if len(expFinder)== 0:
356 self.status = 0
406 self.status = 0
357 return None
407 return None
358
408
359 experimentFilename = os.path.join(experimentCfgPath,expFinder[0])
409 experimentFilename = os.path.join(experimentCfgPath,expFinder[0])
360
410
361 f = open(experimentFilename)
411 f = open(experimentFilename)
362 lines = f.readlines()
412 lines = f.readlines()
363 f.close()
413 f.close()
364
414
365 parmsList = ['npulsesint*','recordsperfile*','nbeamcodes*','ngates*']
415 parmsList = ['npulsesint*','recordsperfile*','nbeamcodes*','ngates*']
366 filterList = [fnmatch.filter(lines, x) for x in parmsList]
416 filterList = [fnmatch.filter(lines, x) for x in parmsList]
367
417
368
418
369 values = [re.sub(r'\D',"",x[0]) for x in filterList]
419 values = [re.sub(r'\D',"",x[0]) for x in filterList]
370
420
371 self.npulsesint_fromfile = int(values[0])
421 self.npulsesint_fromfile = int(values[0])
372 self.recordsperfile_fromfile = int(values[1])
422 self.recordsperfile_fromfile = int(values[1])
373 self.nbeamcodes_fromfile = int(values[2])
423 self.nbeamcodes_fromfile = int(values[2])
374 self.ngates_fromfile = int(values[3])
424 self.ngates_fromfile = int(values[3])
375
425
376 tufileFinder = fnmatch.filter(lines, 'tufile=*')
426 tufileFinder = fnmatch.filter(lines, 'tufile=*')
377 tufile = tufileFinder[0].split('=')[1].split('\n')[0]
427 tufile = tufileFinder[0].split('=')[1].split('\n')[0]
378 tufile = tufile.split('\r')[0]
428 tufile = tufile.split('\r')[0]
379 tufilename = os.path.join(experimentCfgPath,tufile)
429 tufilename = os.path.join(experimentCfgPath,tufile)
380
430
381 f = open(tufilename)
431 f = open(tufilename)
382 lines = f.readlines()
432 lines = f.readlines()
383 f.close()
433 f.close()
384 self.ippSeconds_fromfile = float(lines[1].split()[2])/1E6
434 self.ippSeconds_fromfile = float(lines[1].split()[2])/1E6
385
435
386
436
387 self.status = 1
437 self.status = 1
388
438
389 def __setIdsAndArrays(self):
439 def __setIdsAndArrays(self):
390 self.dataByFrame = self.__setDataByFrame()
440 self.dataByFrame = self.__setDataByFrame()
391 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[0, :]
441 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[0, :]
392 self.readRanges()
442 self.readRanges()
393 self.index_amisr_sample, self.index_amisr_buffer = self.radacHeaderObj.getIndexRangeToPulse(0)
443 self.index_amisr_sample, self.index_amisr_buffer = self.radacHeaderObj.getIndexRangeToPulse(0)
394 self.radacTimeByFrame = numpy.zeros(self.radacHeaderObj.npulses)
444 self.radacTimeByFrame = numpy.zeros(self.radacHeaderObj.npulses)
395 self.buffer_radactime = numpy.zeros_like(self.radacTimeByFrame)
445 if len(self.index_amisr_buffer) > 0:
446 self.buffer_radactime = numpy.zeros_like(self.radacTimeByFrame)
396
447
397
448
398 def __setNextFile(self):
449 def __setNextFile(self,online=False):
399
450
400 newFile = self.__setNextFileOffline()
451 if not(online):
452 newFile = self.__setNextFileOffline()
453 else:
454 newFile = self.__setNextFileOnline()
401
455
402 if not(newFile):
456 if not(newFile):
403 return 0
457 return 0
404
458
405 self.__readHeader()
459 self.__readHeader()
406
460
407 if self.__firstFile:
461 if self.__firstFile:
408 self.__setIdsAndArrays()
462 self.__setIdsAndArrays()
409 self.__firstFile = False
463 self.__firstFile = False
410
464
411 self.__getBeamCode()
465 self.__getBeamCode()
412 self.readDataBlock()
466 self.readDataBlock()
413
467
414
468
415 def setup(self,path=None,
469 def setup(self,path=None,
416 startDate=None,
470 startDate=None,
417 endDate=None,
471 endDate=None,
418 startTime=datetime.time(0,0,0),
472 startTime=datetime.time(0,0,0),
419 endTime=datetime.time(23,59,59),
473 endTime=datetime.time(23,59,59),
420 walk=True,
474 walk=True,
421 timezone='ut',
475 timezone='ut',
422 all=0,):
476 all=0,
477 online=False):
423
478
424 self.timezone = timezone
479 self.timezone = timezone
425 self.all = all
480 self.all = all
426 #Busqueda de archivos offline
481 self.online = online
427 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
482 if not(online):
483 #Busqueda de archivos offline
484 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
485 else:
486 self.__searchFilesOnline(path, walk)
428
487
429 if not(self.filenameList):
488 if not(self.filenameList):
430 print "There is no files into the folder: %s"%(path)
489 print "There is no files into the folder: %s"%(path)
431
490
432 sys.exit(-1)
491 sys.exit(-1)
433
492
434 self.__getExpParameters()
493 self.__getExpParameters()
435
494
436 self.fileIndex = -1
495 self.fileIndex = -1
437
496
438 self.__setNextFile()
497 self.__setNextFile(online)
439
498
440 # first_beamcode = self.radacHeaderObj.beamCodeByPulse[0,0]
499 # first_beamcode = self.radacHeaderObj.beamCodeByPulse[0,0]
441 # index = numpy.where(self.radacHeaderObj.beamCodeByPulse[0,:]!=first_beamcode)[0][0]
500 # index = numpy.where(self.radacHeaderObj.beamCodeByPulse[0,:]!=first_beamcode)[0][0]
442 self.profileIndex_offset = self.radacHeaderObj.pulseCount[0,:][0]
501 self.profileIndex_offset = self.radacHeaderObj.pulseCount[0,:][0]
443 self.profileIndex = self.profileIndex_offset
502 self.profileIndex = self.profileIndex_offset
444
503
445 def readRanges(self):
504 def readRanges(self):
446 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range')
505 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range')
447
506
448 self.rangeFromFile = numpy.reshape(dataset.value,(-1))
507 self.rangeFromFile = numpy.reshape(dataset.value,(-1))
449 return self.rangeFromFile
508 return self.rangeFromFile
450
509
451
510
452 def readRadacTime(self,idrecord, range1, range2):
511 def readRadacTime(self,idrecord, range1, range2):
453 self.radacTimeFromFile = self.radacHeaderObj.radacTime.value
512 self.radacTimeFromFile = self.radacHeaderObj.radacTime.value
454
513
455 radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
514 radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
456 #radacTimeByFrame = dataset[idrecord - 1,range1]
515 #radacTimeByFrame = dataset[idrecord - 1,range1]
457 #radacTimeByFrame = dataset[idrecord,range2]
516 #radacTimeByFrame = dataset[idrecord,range2]
458
517
459 return radacTimeByFrame
518 return radacTimeByFrame
460
519
461 def readBeamCode(self, idrecord, range1, range2):
520 def readBeamCode(self, idrecord, range1, range2):
462 dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode')
521 dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode')
463 beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
522 beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
464 self.beamCodesFromFile = dataset.value
523 self.beamCodesFromFile = dataset.value
465
524
466 #beamcodeByFrame[range1] = dataset[idrecord - 1, range1]
525 #beamcodeByFrame[range1] = dataset[idrecord - 1, range1]
467 #beamcodeByFrame[range2] = dataset[idrecord, range2]
526 #beamcodeByFrame[range2] = dataset[idrecord, range2]
468 beamcodeByFrame[range1] = dataset[idrecord, range1]
527 beamcodeByFrame[range1] = dataset[idrecord, range1]
469 beamcodeByFrame[range2] = dataset[idrecord, range2]
528 beamcodeByFrame[range2] = dataset[idrecord, range2]
470
529
471 return beamcodeByFrame
530 return beamcodeByFrame
472
531
473
532
474 def __setDataByFrame(self):
533 def __setDataByFrame(self):
475 ndata = 2 # porque es complejo
534 ndata = 2 # porque es complejo
476 dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata))
535 dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata))
477 return dataByFrame
536 return dataByFrame
478
537
479 def __readDataSet(self):
538 def __readDataSet(self):
480 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
539 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
481 return dataset
540 return dataset
482
541
483 def __setDataBlock(self,):
542 def __setDataBlock(self,):
484 real = self.dataByFrame[:,:,0] #asumo que 0 es real
543 real = self.dataByFrame[:,:,0] #asumo que 0 es real
485 imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario
544 imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario
486 datablock = real + imag*1j #armo el complejo
545 datablock = real + imag*1j #armo el complejo
487 return datablock
546 return datablock
488
547
489 def readSamples_version1(self,idrecord):
548 def readSamples_version1(self,idrecord):
490 #estas tres primeras lineas solo se deben ejecutar una vez
549 #estas tres primeras lineas solo se deben ejecutar una vez
491 if self.flagIsNewFile:
550 if self.flagIsNewFile:
492 #reading dataset
551 #reading dataset
493 self.dataset = self.__readDataSet()
552 self.dataset = self.__readDataSet()
494 self.flagIsNewFile = 0
553 self.flagIsNewFile = 0
495
554
496 if idrecord == 0:
555 if idrecord == 0:
497 self.dataByFrame[self.index4_schain_datablock, : ,:] = self.dataset[0, self.index_amisr_sample,:,:]
556 self.dataByFrame[self.index4_schain_datablock, : ,:] = self.dataset[0, self.index_amisr_sample,:,:]
498 self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[0, self.index_amisr_sample]
557 self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[0, self.index_amisr_sample]
499 datablock = self.__setDataBlock()
558 datablock = self.__setDataBlock()
500
559 if len(self.index_amisr_buffer) > 0:
501 self.buffer = self.dataset[0, self.index_amisr_buffer,:,:]
560 self.buffer = self.dataset[0, self.index_amisr_buffer,:,:]
502 self.buffer_radactime = self.radacHeaderObj.radacTime[0, self.index_amisr_buffer]
561 self.buffer_radactime = self.radacHeaderObj.radacTime[0, self.index_amisr_buffer]
503
562
504 return datablock
563 return datablock
505
564 if len(self.index_amisr_buffer) > 0:
506 self.dataByFrame[self.index4_buffer,:,:] = self.buffer.copy()
565 self.dataByFrame[self.index4_buffer,:,:] = self.buffer.copy()
507 self.radacTimeByFrame[self.index4_buffer] = self.buffer_radactime.copy()
566 self.radacTimeByFrame[self.index4_buffer] = self.buffer_radactime.copy()
508 self.dataByFrame[self.index4_schain_datablock,:,:] = self.dataset[idrecord, self.index_amisr_sample,:,:]
567 self.dataByFrame[self.index4_schain_datablock,:,:] = self.dataset[idrecord, self.index_amisr_sample,:,:]
509 self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_sample]
568 self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_sample]
510 datablock = self.__setDataBlock()
569 datablock = self.__setDataBlock()
511
570 if len(self.index_amisr_buffer) > 0:
512 self.buffer = self.dataset[idrecord, self.index_amisr_buffer, :, :]
571 self.buffer = self.dataset[idrecord, self.index_amisr_buffer, :, :]
513 self.buffer_radactime = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_buffer]
572 self.buffer_radactime = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_buffer]
514
573
515 return datablock
574 return datablock
516
575
517
576
518 def readSamples(self,idrecord):
577 def readSamples(self,idrecord):
519 if self.flagIsNewFile:
578 if self.flagIsNewFile:
520 self.dataByFrame = self.__setDataByFrame()
579 self.dataByFrame = self.__setDataByFrame()
521 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :]
580 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :]
522
581
523 #reading ranges
582 #reading ranges
524 self.readRanges()
583 self.readRanges()
525 #reading dataset
584 #reading dataset
526 self.dataset = self.__readDataSet()
585 self.dataset = self.__readDataSet()
527
586
528 self.flagIsNewFile = 0
587 self.flagIsNewFile = 0
529 self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :]
588 self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :]
530 self.dataByFrame = self.dataset[idrecord, :, :, :]
589 self.dataByFrame = self.dataset[idrecord, :, :, :]
531 datablock = self.__setDataBlock()
590 datablock = self.__setDataBlock()
532 return datablock
591 return datablock
533
592
534
593
535 def readDataBlock(self):
594 def readDataBlock(self):
536
595
537 self.datablock = self.readSamples_version1(self.idrecord_count)
596 self.datablock = self.readSamples_version1(self.idrecord_count)
538 #self.datablock = self.readSamples(self.idrecord_count)
597 #self.datablock = self.readSamples(self.idrecord_count)
539 #print 'record:', self.idrecord_count
598 #print 'record:', self.idrecord_count
540
599
541 self.idrecord_count += 1
600 self.idrecord_count += 1
542 self.profileIndex = 0
601 self.profileIndex = 0
543
602
544 if self.idrecord_count >= self.radacHeaderObj.nrecords:
603 if self.idrecord_count >= self.radacHeaderObj.nrecords:
545 self.idrecord_count = 0
604 self.idrecord_count = 0
546 self.flagIsNewFile = 1
605 self.flagIsNewFile = 1
547
606
548 def readNextBlock(self):
607 def readNextBlock(self):
549
608
550 self.readDataBlock()
609 self.readDataBlock()
551
610
552 if self.flagIsNewFile:
611 if self.flagIsNewFile:
553 self.__setNextFile()
612 self.__setNextFile(self.online)
554 pass
613 pass
555
614
556 def __hasNotDataInBuffer(self):
615 def __hasNotDataInBuffer(self):
557 #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record
616 #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record
558 if self.profileIndex >= self.radacHeaderObj.npulses:
617 if self.profileIndex >= self.radacHeaderObj.npulses:
559 return 1
618 return 1
560 return 0
619 return 0
561
620
562 def printUTC(self):
621 def printUTC(self):
563 print self.dataOut.utctime
622 print self.dataOut.utctime
564 print ''
623 print ''
565
624
566 def setObjProperties(self):
625 def setObjProperties(self):
567 self.dataOut.heightList = self.rangeFromFile/1000.0 #km
626 self.dataOut.heightList = self.rangeFromFile/1000.0 #km
568 self.dataOut.nProfiles = self.radacHeaderObj.npulses
627 self.dataOut.nProfiles = self.radacHeaderObj.npulses
569 self.dataOut.nRecords = self.radacHeaderObj.nrecords
628 self.dataOut.nRecords = self.radacHeaderObj.nrecords
570 self.dataOut.nBeams = self.radacHeaderObj.nbeams
629 self.dataOut.nBeams = self.radacHeaderObj.nbeams
571 self.dataOut.ippSeconds = self.ippSeconds_fromfile
630 self.dataOut.ippSeconds = self.ippSeconds_fromfile
572 self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
631 self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
573 self.dataOut.frequency = self.frequency_h5file
632 self.dataOut.frequency = self.frequency_h5file
574 self.dataOut.npulseByFrame = self.npulseByFrame
633 self.dataOut.npulseByFrame = self.npulseByFrame
575 self.dataOut.nBaud = None
634 self.dataOut.nBaud = None
576 self.dataOut.nCode = None
635 self.dataOut.nCode = None
577 self.dataOut.code = None
636 self.dataOut.code = None
578
637
579 self.dataOut.beamCodeDict = self.beamCodeDict
638 self.dataOut.beamCodeDict = self.beamCodeDict
580 self.dataOut.beamRangeDict = self.beamRangeDict
639 self.dataOut.beamRangeDict = self.beamRangeDict
581
640
582 if self.timezone == 'lt':
641 if self.timezone == 'lt':
583 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
642 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
584 else:
643 else:
585 self.dataOut.timeZone = 0 #by default time is UTC
644 self.dataOut.timeZone = 0 #by default time is UTC
586
645
587 def getData(self):
646 def getData(self):
588
647
589 if self.flagNoMoreFiles:
648 if self.flagNoMoreFiles:
590 self.dataOut.flagNoData = True
649 self.dataOut.flagNoData = True
591 print 'Process finished'
650 print 'Process finished'
592 return 0
651 return 0
593
652
594 if self.__hasNotDataInBuffer():
653 if self.__hasNotDataInBuffer():
595 self.readNextBlock()
654 self.readNextBlock()
596
655
597
656
598 if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
657 if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
599 self.dataOut.flagNoData = True
658 self.dataOut.flagNoData = True
600 return 0
659 return 0
601
660
602 self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1))
661 self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1))
603
662
604 self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex]
663 self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex]
605 self.dataOut.profileIndex = self.profileIndex
664 self.dataOut.profileIndex = self.profileIndex
606 self.dataOut.flagNoData = False
665 self.dataOut.flagNoData = False
607
666
608 self.profileIndex += 1
667 self.profileIndex += 1
609
668
610 return self.dataOut.data
669 return self.dataOut.data
611
670
612
671
613 def run(self, **kwargs):
672 def run(self, **kwargs):
614 if not(self.isConfig):
673 if not(self.isConfig):
615 self.setup(**kwargs)
674 self.setup(**kwargs)
616 self.setObjProperties()
675 self.setObjProperties()
617 self.isConfig = True
676 self.isConfig = True
618
677
619 self.getData()
678 self.getData()
General Comments 0
You need to be logged in to leave comments. Login now