##// END OF EJS Templates
schain
RSS
algunos cambios
Jose Chavez -
r982:9c41836094db
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,661 +1,676
1
1
2 '''
2 '''
3 Created on Jul 3, 2014
3 Created on Jul 3, 2014
4
4
5 @author: roj-idl71
5 @author: roj-idl71
6 '''
6 '''
7 import os
7 import os
8 import datetime
8 import datetime
9 import numpy
9 import numpy
10 from profilehooks import coverage, profile
10 from profilehooks import coverage, profile
11 from fractions import Fraction
11 from fractions import Fraction
12
12
13 try:
13 try:
14 from gevent import sleep
14 from gevent import sleep
15 except:
15 except:
16 from time import sleep
16 from time import sleep
17
17
18 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
18 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
19 from schainpy.model.data.jrodata import Voltage
19 from schainpy.model.data.jrodata import Voltage
20 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
20 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
21 from time import time
21 from time import time
22 import cPickle
22 import cPickle
23 try:
23 try:
24 import digital_rf
24 import digital_rf
25 except:
25 except:
26 print 'You should install "digital_rf" module if you want to read Digital RF data'
26 print 'You should install "digital_rf" module if you want to read Digital RF data'
27
27
28 class DigitalRFReader(ProcessingUnit):
28 class DigitalRFReader(ProcessingUnit):
29 '''
29 '''
30 classdocs
30 classdocs
31 '''
31 '''
32
32
33 def __init__(self, **kwargs):
33 def __init__(self, **kwargs):
34 '''
34 '''
35 Constructor
35 Constructor
36 '''
36 '''
37
37
38 ProcessingUnit.__init__(self, **kwargs)
38 ProcessingUnit.__init__(self, **kwargs)
39
39
40 self.dataOut = Voltage()
40 self.dataOut = Voltage()
41 self.__printInfo = True
41 self.__printInfo = True
42 self.__flagDiscontinuousBlock = False
42 self.__flagDiscontinuousBlock = False
43 self.__bufferIndex = 9999999
43 self.__bufferIndex = 9999999
44 self.__ippKm = None
44 self.__ippKm = None
45 self.__codeType = 0
45 self.__codeType = 0
46 self.__nCode = None
46 self.__nCode = None
47 self.__nBaud = None
47 self.__nBaud = None
48 self.__code = None
48 self.__code = None
49
49
50 def __getCurrentSecond(self):
50 def __getCurrentSecond(self):
51
51
52 return self.__thisUnixSample/self.__sample_rate
52 return self.__thisUnixSample/self.__sample_rate
53
53
54 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
54 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
55
55
56 def __setFileHeader(self):
56 def __setFileHeader(self):
57 '''
57 '''
58 In this method will be initialized every parameter of dataOut object (header, no data)
58 In this method will be initialized every parameter of dataOut object (header, no data)
59 '''
59 '''
60 ippSeconds = 1.0*self.__nSamples/self.__sample_rate
60 ippSeconds = 1.0*self.__nSamples/self.__sample_rate
61
61
62 nProfiles = 1.0/ippSeconds # Number of profiles in one second
62 nProfiles = 1.0/ippSeconds # Number of profiles in one second
63
63
64 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(self.__radarControllerHeader)
64 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(self.__radarControllerHeader)
65
65
66 self.dataOut.systemHeaderObj = SystemHeader(self.__systemHeader)
66 self.dataOut.systemHeaderObj = SystemHeader(self.__systemHeader)
67
67
68 self.dataOut.type = "Voltage"
68 self.dataOut.type = "Voltage"
69
69
70 self.dataOut.data = None
70 self.dataOut.data = None
71
71
72 self.dataOut.dtype = self.dtype
72 self.dataOut.dtype = self.dtype
73
73
74 # self.dataOut.nChannels = 0
74 # self.dataOut.nChannels = 0
75
75
76 # self.dataOut.nHeights = 0
76 # self.dataOut.nHeights = 0
77
77
78 self.dataOut.nProfiles = nProfiles
78 self.dataOut.nProfiles = nProfiles
79
79
80 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
80 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
81
81
82 self.dataOut.channelList = self.__channelList
82 self.dataOut.channelList = self.__channelList
83
83
84 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
84 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
85
85
86 # self.dataOut.channelIndexList = None
86 # self.dataOut.channelIndexList = None
87
87
88 self.dataOut.flagNoData = True
88 self.dataOut.flagNoData = True
89
89 self.dataOut.flagDataAsBlock = False
90 self.dataOut.flagDataAsBlock = False
90 # Set to TRUE if the data is discontinuous
91 # Set to TRUE if the data is discontinuous
91 self.dataOut.flagDiscontinuousBlock = False
92 self.dataOut.flagDiscontinuousBlock = False
92
93
93 self.dataOut.utctime = None
94 self.dataOut.utctime = None
94
95
95 self.dataOut.timeZone = self.__timezone/60 # timezone like jroheader, difference in minutes between UTC and localtime
96 self.dataOut.timeZone = self.__timezone/60 # timezone like jroheader, difference in minutes between UTC and localtime
96
97
97 self.dataOut.dstFlag = 0
98 self.dataOut.dstFlag = 0
98
99
99 self.dataOut.errorCount = 0
100 self.dataOut.errorCount = 0
100
101
101 self.dataOut.nCohInt = 1
102 self.dataOut.nCohInt = self.fixed_metadata_dict['nCohInt']
103
104 self.dataOut.flagDecodeData = self.fixed_metadata_dict['flagDecodeData'] # asumo que la data esta decodificada
102
105
103 self.dataOut.flagDecodeData = False # asumo que la data esta decodificada
106 self.dataOut.flagDeflipData = self.fixed_metadata_dict['flagDeflipData'] # asumo que la data esta sin flip
104
107
105 self.dataOut.flagDeflipData = False # asumo que la data esta sin flip
108 self.dataOut.flagShiftFFT = self.fixed_metadata_dict['flagShiftFFT']
106
109
107 self.dataOut.flagShiftFFT = False
110 self.dataOut.useLocalTime = self.fixed_metadata_dict['useLocalTime']
108
111
109 self.dataOut.ippSeconds = ippSeconds
112 self.dataOut.ippSeconds = ippSeconds
110
113
111 # Time interval between profiles
114 # Time interval between profiles
112 # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
115 # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
113
116
114 self.dataOut.frequency = self.__frequency
117 self.dataOut.frequency = self.__frequency
115
118
116 self.dataOut.realtime = self.__online
119 self.dataOut.realtime = self.__online
117
120
118 def findDatafiles(self, path, startDate=None, endDate=None):
121 def findDatafiles(self, path, startDate=None, endDate=None):
119
122
120 if not os.path.isdir(path):
123 if not os.path.isdir(path):
121 return []
124 return []
122
125
123 try:
126 try:
124 digitalReadObj = digital_rf.DigitalRFReader(path, load_all_metadata=True)
127 digitalReadObj = digital_rf.DigitalRFReader(path, load_all_metadata=True)
125 except:
128 except:
126 digitalReadObj = digital_rf.DigitalRFReader(path)
129 digitalReadObj = digital_rf.DigitalRFReader(path)
127
130
128 channelNameList = digitalReadObj.get_channels()
131 channelNameList = digitalReadObj.get_channels()
129
132
130 if not channelNameList:
133 if not channelNameList:
131 return []
134 return []
132
135
133 metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
136 metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
134
137
135 sample_rate = metadata_dict['sample_rate'][0]
138 sample_rate = metadata_dict['sample_rate'][0]
136
139
137 this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
140 this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
138
141
139 try:
142 try:
140 timezone = this_metadata_file['timezone'].value
143 timezone = this_metadata_file['timezone'].value
141 except:
144 except:
142 timezone = 0
145 timezone = 0
143
146
144 startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(channelNameList[0])/sample_rate - timezone
147 startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(channelNameList[0])/sample_rate - timezone
145
148
146 startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
149 startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
147 endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
150 endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
148
151
149 if not startDate:
152 if not startDate:
150 startDate = startDatetime.date()
153 startDate = startDatetime.date()
151
154
152 if not endDate:
155 if not endDate:
153 endDate = endDatatime.date()
156 endDate = endDatatime.date()
154
157
155 dateList = []
158 dateList = []
156
159
157 thisDatetime = startDatetime
160 thisDatetime = startDatetime
158
161
159 while(thisDatetime<=endDatatime):
162 while(thisDatetime<=endDatatime):
160
163
161 thisDate = thisDatetime.date()
164 thisDate = thisDatetime.date()
162
165
163 if thisDate < startDate:
166 if thisDate < startDate:
164 continue
167 continue
165
168
166 if thisDate > endDate:
169 if thisDate > endDate:
167 break
170 break
168
171
169 dateList.append(thisDate)
172 dateList.append(thisDate)
170 thisDatetime += datetime.timedelta(1)
173 thisDatetime += datetime.timedelta(1)
171
174
172 return dateList
175 return dateList
173
176
174 def setup(self, path = None,
177 def setup(self, path = None,
175 startDate = None,
178 startDate = None,
176 endDate = None,
179 endDate = None,
177 startTime = datetime.time(0,0,0),
180 startTime = datetime.time(0,0,0),
178 endTime = datetime.time(23,59,59),
181 endTime = datetime.time(23,59,59),
179 channelList = None,
182 channelList = None,
180 nSamples = None,
183 nSamples = None,
181 ippKm = 60,
182 online = False,
184 online = False,
183 delay = 60,
185 delay = 60,
184 buffer_size = 1024,
186 buffer_size = 1024,
187 ippKm=None,
185 **kwargs):
188 **kwargs):
186 '''
189 '''
187 In this method we should set all initial parameters.
190 In this method we should set all initial parameters.
188
191
189 Inputs:
192 Inputs:
190 path
193 path
191 startDate
194 startDate
192 endDate
195 endDate
193 startTime
196 startTime
194 endTime
197 endTime
195 set
198 set
196 expLabel
199 expLabel
197 ext
200 ext
198 online
201 online
199 delay
202 delay
200 '''
203 '''
201
204
202 if not os.path.isdir(path):
205 if not os.path.isdir(path):
203 raise ValueError, "[Reading] Directory %s does not exist" %path
206 raise ValueError, "[Reading] Directory %s does not exist" %path
204
207
205 try:
208 try:
206 self.digitalReadObj = digital_rf.DigitalRFReader(path, load_all_metadata=True)
209 self.digitalReadObj = digital_rf.DigitalRFReader(path, load_all_metadata=True)
207 except:
210 except:
208 self.digitalReadObj = digital_rf.DigitalRFReader(path)
211 self.digitalReadObj = digital_rf.DigitalRFReader(path)
209
212
210 channelNameList = self.digitalReadObj.get_channels()
213 channelNameList = self.digitalReadObj.get_channels()
211
214
212 if not channelNameList:
215 if not channelNameList:
213 raise ValueError, "[Reading] Directory %s does not have any files" %path
216 raise ValueError, "[Reading] Directory %s does not have any files" %path
214
217
215 if not channelList:
218 if not channelList:
216 channelList = range(len(channelNameList))
219 channelList = range(len(channelNameList))
217
220
218
221
219 ########## Reading metadata ######################
222 ########## Reading metadata ######################
220
223
221 top_properties = self.digitalReadObj.get_properties(channelNameList[channelList[0]])
224 top_properties = self.digitalReadObj.get_properties(channelNameList[channelList[0]])
222
225
223 self.__sample_rate = 1.0 * top_properties['sample_rate_numerator'] / top_properties['sample_rate_denominator']
226 self.__sample_rate = 1.0 * top_properties['sample_rate_numerator'] / top_properties['sample_rate_denominator']
227
224 # self.__samples_per_file = top_properties['samples_per_file'][0]
228 # self.__samples_per_file = top_properties['samples_per_file'][0]
225 self.__deltaHeigth = 1e6*0.15/self.__sample_rate ## why 0.15?
229 self.__deltaHeigth = 1e6*0.15/self.__sample_rate ## why 0.15?
226
230
227 this_metadata_file = self.digitalReadObj.get_digital_metadata(channelNameList[channelList[0]])
231 this_metadata_file = self.digitalReadObj.get_digital_metadata(channelNameList[channelList[0]])
228 metadata_bounds = this_metadata_file.get_bounds()
232 metadata_bounds = this_metadata_file.get_bounds()
229 self.fixed_metadata_dict = this_metadata_file.read(metadata_bounds[0])[metadata_bounds[0]] ## GET FIRST HEADER
233 self.fixed_metadata_dict = this_metadata_file.read(metadata_bounds[0])[metadata_bounds[0]] ## GET FIRST HEADER
230 self.__processingHeader = self.fixed_metadata_dict['processingHeader']
234 self.__processingHeader = self.fixed_metadata_dict['processingHeader']
231 self.__radarControllerHeader = self.fixed_metadata_dict['radarControllerHeader']
235 self.__radarControllerHeader = self.fixed_metadata_dict['radarControllerHeader']
232 self.__systemHeader = self.fixed_metadata_dict['systemHeader']
236 self.__systemHeader = self.fixed_metadata_dict['systemHeader']
233 self.dtype = cPickle.loads(self.fixed_metadata_dict['dtype'])
237 self.dtype = cPickle.loads(self.fixed_metadata_dict['dtype'])
234
238
235 self.__frequency = None
239 self.__frequency = None
236
240
237 try:
241 try:
238 self.__frequency = self.fixed_metadata_dict['frequency']
242 self.__frequency = self.fixed_metadata_dict['frequency']
239 except:
243 except:
240 self.__frequency = None
244 self.__frequency = None
241
245
242 try:
246 try:
243 self.__timezone = self.fixed_metadata_dict['timezone']
247 self.__timezone = self.fixed_metadata_dict['timezone'] * 60
244 except:
248 except:
245 self.__timezone = 0
249 self.__timezone = 0
246
250
247
251
248 try:
252 try:
249 nSamples = self.__systemHeader['nSamples']
253 nSamples = self.fixed_metadata_dict['nSamples']
250 except:
254 except:
251 nSamples = None
255 nSamples = None
252
256
253 self.__firstHeigth = 0
257 self.__firstHeigth = 0
254
258
255 try:
259 try:
256 codeType = self.__radarControllerHeader['codeType']
260 codeType = self.__radarControllerHeader['codeType']
257 except:
261 except:
258 codeType = 0
262 codeType = 0
259
263
260 nCode = 1
264 nCode = 1
261 nBaud = 1
265 nBaud = 1
262 code = numpy.ones((nCode, nBaud), dtype=numpy.int)
266 code = numpy.ones((nCode, nBaud), dtype=numpy.int)
263
267
264 if codeType:
268 if codeType:
265 nCode = self.__radarControllerHeader['nCode']
269 nCode = self.__radarControllerHeader['nCode']
266 nBaud = self.__radarControllerHeader['nBaud']
270 nBaud = self.__radarControllerHeader['nBaud']
267 code = self.__radarControllerHeader['code']
271 code = self.__radarControllerHeader['code']
268
272
269 if not ippKm:
273 if not ippKm:
270 try:
274 try:
271 # seconds to km
275 # seconds to km
272 ippKm = 1e6*0.15*self.__radarControllerHeader['ipp']
276 ippKm = self.__radarControllerHeader['ipp']
273 except:
277 except:
274 ippKm = None
278 ippKm = None
275 ####################################################
279 ####################################################
276 startUTCSecond = None
280 startUTCSecond = None
277 endUTCSecond = None
281 endUTCSecond = None
278
282
279 if startDate:
283 if startDate:
280 startDatetime = datetime.datetime.combine(startDate, startTime)
284 startDatetime = datetime.datetime.combine(startDate, startTime)
281 startUTCSecond = (startDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
285 startUTCSecond = (startDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
282
286
283 if endDate:
287 if endDate:
284 endDatetime = datetime.datetime.combine(endDate, endTime)
288 endDatetime = datetime.datetime.combine(endDate, endTime)
285 endUTCSecond = (endDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
289 endUTCSecond = (endDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
286
290
287 start_index, end_index = self.digitalReadObj.get_bounds(channelNameList[channelList[0]])
291 start_index, end_index = self.digitalReadObj.get_bounds(channelNameList[channelList[0]])
288
292
289 if not startUTCSecond:
293 if not startUTCSecond:
290 startUTCSecond = start_index/self.__sample_rate
294 startUTCSecond = start_index/self.__sample_rate
291
295
292 if start_index > startUTCSecond*self.__sample_rate:
296 if start_index > startUTCSecond*self.__sample_rate:
293 startUTCSecond = start_index/self.__sample_rate
297 startUTCSecond = start_index/self.__sample_rate
294
298
295 if not endUTCSecond:
299 if not endUTCSecond:
296 endUTCSecond = end_index/self.__sample_rate
300 endUTCSecond = end_index/self.__sample_rate
297
301
298 if end_index < endUTCSecond*self.__sample_rate:
302 if end_index < endUTCSecond*self.__sample_rate:
299 endUTCSecond = end_index/self.__sample_rate
303 endUTCSecond = end_index/self.__sample_rate
300
304 print ippKm
301 if not nSamples:
305 if not nSamples:
302 if not ippKm:
306 if not ippKm:
303 raise ValueError, "[Reading] nSamples or ippKm should be defined"
307 raise ValueError, "[Reading] nSamples or ippKm should be defined"
304 nSamples = int(ippKm / (1e6*0.15/self.__sample_rate))
308 nSamples = int(ippKm / (1e6*0.15/self.__sample_rate))
305
309 print nSamples
306 channelBoundList = []
310 channelBoundList = []
307 channelNameListFiltered = []
311 channelNameListFiltered = []
308
312
309 for thisIndexChannel in channelList:
313 for thisIndexChannel in channelList:
310 thisChannelName = channelNameList[thisIndexChannel]
314 thisChannelName = channelNameList[thisIndexChannel]
311 start_index, end_index = self.digitalReadObj.get_bounds(thisChannelName)
315 start_index, end_index = self.digitalReadObj.get_bounds(thisChannelName)
312 channelBoundList.append((start_index, end_index))
316 channelBoundList.append((start_index, end_index))
313 channelNameListFiltered.append(thisChannelName)
317 channelNameListFiltered.append(thisChannelName)
314
318
315 self.profileIndex = 0
319 self.profileIndex = 0
316 self.i= 0
320 self.i= 0
317 self.__delay = delay
321 self.__delay = delay
318 self.__ippKm = ippKm
322 self.__ippKm = ippKm
319 self.__codeType = codeType
323 self.__codeType = codeType
320 self.__nCode = nCode
324 self.__nCode = nCode
321 self.__nBaud = nBaud
325 self.__nBaud = nBaud
322 self.__code = code
326 self.__code = code
323
327
324 self.__datapath = path
328 self.__datapath = path
325 self.__online = online
329 self.__online = online
326 self.__channelList = channelList
330 self.__channelList = channelList
327 self.__channelNameList = channelNameListFiltered
331 self.__channelNameList = channelNameListFiltered
328 self.__channelBoundList = channelBoundList
332 self.__channelBoundList = channelBoundList
329 self.__nSamples = nSamples
333 self.__nSamples = nSamples
330 self.__samples_to_read = long(nSamples) # FIJO: AHORA 40
334 self.__samples_to_read = long(nSamples) # FIJO: AHORA 40
331 self.__nChannels = len(self.__channelList)
335 self.__nChannels = len(self.__channelList)
332
336
333 self.__startUTCSecond = startUTCSecond
337 self.__startUTCSecond = startUTCSecond
334 self.__endUTCSecond = endUTCSecond
338 self.__endUTCSecond = endUTCSecond
335
339
336 self.__timeInterval = 1.0 * self.__samples_to_read/self.__sample_rate # Time interval
340 self.__timeInterval = 1.0 * self.__samples_to_read/self.__sample_rate # Time interval
337
341
338 if online:
342 if online:
339 # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
343 # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
340 startUTCSecond = numpy.floor(endUTCSecond)
344 startUTCSecond = numpy.floor(endUTCSecond)
341
345
342 self.__thisUnixSample = long(startUTCSecond*self.__sample_rate) - self.__samples_to_read ## por que en el otro metodo lo primero q se hace es sumar samplestoread
346 self.__thisUnixSample = long(startUTCSecond*self.__sample_rate) - self.__samples_to_read ## por que en el otro metodo lo primero q se hace es sumar samplestoread
343
347
344 self.__data_buffer = numpy.zeros((self.__nChannels, self.__samples_to_read), dtype = numpy.complex)
348 self.__data_buffer = numpy.zeros((self.__nChannels, self.__samples_to_read), dtype = numpy.complex)
345
349
346 self.__setFileHeader()
350 self.__setFileHeader()
347 self.isConfig = True
351 self.isConfig = True
348
352
349 print "[Reading] Digital RF Data was found from %s to %s " %(
353 print "[Reading] Digital RF Data was found from %s to %s " %(
350 datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
354 datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
351 datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
355 datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
352 )
356 )
353
357
354 print "[Reading] Starting process from %s to %s" %(datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
358 print "[Reading] Starting process from %s to %s" %(datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
355 datetime.datetime.utcfromtimestamp(endUTCSecond - self.__timezone)
359 datetime.datetime.utcfromtimestamp(endUTCSecond - self.__timezone)
356 )
360 )
357
361
358 def __reload(self):
362 def __reload(self):
359 # print
363 # print
360 # print "%s not in range [%s, %s]" %(
364 # print "%s not in range [%s, %s]" %(
361 # datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
365 # datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
362 # datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
366 # datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
363 # datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
367 # datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
364 # )
368 # )
365 print "[Reading] reloading metadata ..."
369 print "[Reading] reloading metadata ..."
366
370
367 try:
371 try:
368 self.digitalReadObj.reload(complete_update=True)
372 self.digitalReadObj.reload(complete_update=True)
369 except:
373 except:
370 self.digitalReadObj.reload()
374 self.digitalReadObj.reload()
371
375
372 start_index, end_index = self.digitalReadObj.get_bounds(self.__channelNameList[self.__channelList[0]])
376 start_index, end_index = self.digitalReadObj.get_bounds(self.__channelNameList[self.__channelList[0]])
373
377
374 if start_index > self.__startUTCSecond*self.__sample_rate:
378 if start_index > self.__startUTCSecond*self.__sample_rate:
375 self.__startUTCSecond = 1.0*start_index/self.__sample_rate
379 self.__startUTCSecond = 1.0*start_index/self.__sample_rate
376
380
377 if end_index > self.__endUTCSecond*self.__sample_rate:
381 if end_index > self.__endUTCSecond*self.__sample_rate:
378 self.__endUTCSecond = 1.0*end_index/self.__sample_rate
382 self.__endUTCSecond = 1.0*end_index/self.__sample_rate
379 print
383 print
380 print "[Reading] New timerange found [%s, %s] " %(
384 print "[Reading] New timerange found [%s, %s] " %(
381 datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
385 datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
382 datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
386 datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
383 )
387 )
384
388
385 return True
389 return True
386
390
387 return False
391 return False
388
392
389 def __readNextBlock(self, seconds=30, volt_scale = 1):
393 def __readNextBlock(self, seconds=30, volt_scale = 1):
390 '''
394 '''
391 '''
395 '''
392
396
393 # Set the next data
397 # Set the next data
394 self.__flagDiscontinuousBlock = False
398 self.__flagDiscontinuousBlock = False
395 self.__thisUnixSample += self.__samples_to_read
399 self.__thisUnixSample += self.__samples_to_read
396
400
397 if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
401 if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
398 print "[Reading] There are no more data into selected time-range"
402 print "[Reading] There are no more data into selected time-range"
399 if self.__online:
403 if self.__online:
400 self.__reload()
404 self.__reload()
401 else:
405 else:
402 return False
406 return False
403
407
404 if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
408 if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
405 return False
409 return False
406 self.__thisUnixSample -= self.__samples_to_read
410 self.__thisUnixSample -= self.__samples_to_read
407
411
408 indexChannel = 0
412 indexChannel = 0
409
413
410 dataOk = False
414 dataOk = False
411 for thisChannelName in self.__channelNameList: ##TODO VARIOS CHANNELS?
415 for thisChannelName in self.__channelNameList: ##TODO VARIOS CHANNELS?
412 try:
416 try:
413 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
417 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
414 self.__samples_to_read,
418 self.__samples_to_read,
415 thisChannelName)
419 thisChannelName)
416 except IOError, e:
420 except IOError, e:
417 #read next profile
421 #read next profile
418 self.__flagDiscontinuousBlock = True
422 self.__flagDiscontinuousBlock = True
419 print "[Reading] %s" %datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e
423 print "[Reading] %s" %datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e
420 break
424 break
421
425
422 if result.shape[0] != self.__samples_to_read:
426 if result.shape[0] != self.__samples_to_read:
423 self.__flagDiscontinuousBlock = True
427 self.__flagDiscontinuousBlock = True
424 print "[Reading] %s: Too few samples were found, just %d/%d samples" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
428 print "[Reading] %s: Too few samples were found, just %d/%d samples" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
425 result.shape[0],
429 result.shape[0],
426 self.__samples_to_read)
430 self.__samples_to_read)
427 break
431 break
428
432
429 self.__data_buffer[indexChannel,:] = result*volt_scale
433 self.__data_buffer[indexChannel,:] = result*volt_scale
430
434
431 indexChannel += 1
435 indexChannel += 1
432
436
433 dataOk = True
437 dataOk = True
434
438
435 self.__utctime = self.__thisUnixSample/self.__sample_rate
439 self.__utctime = self.__thisUnixSample/self.__sample_rate
436
440
437 if not dataOk:
441 if not dataOk:
438 return False
442 return False
439
443
440 print "[Reading] %s: %d samples <> %f sec" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
444 print "[Reading] %s: %d samples <> %f sec" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
441 self.__samples_to_read,
445 self.__samples_to_read,
442 self.__timeInterval)
446 self.__timeInterval)
443
447
444 self.__bufferIndex = 0
448 self.__bufferIndex = 0
445
449
446 return True
450 return True
447
451
448 def __isBufferEmpty(self):
452 def __isBufferEmpty(self):
449 return self.__bufferIndex > self.__samples_to_read - self.__nSamples #40960 - 40
453 return self.__bufferIndex > self.__samples_to_read - self.__nSamples #40960 - 40
450
454
451 def getData(self, seconds=30, nTries=5):
455 def getData(self, seconds=30, nTries=5):
452
456
453 '''
457 '''
454 This method gets the data from files and put the data into the dataOut object
458 This method gets the data from files and put the data into the dataOut object
455
459
456 In addition, increase el the buffer counter in one.
460 In addition, increase el the buffer counter in one.
457
461
458 Return:
462 Return:
459 data : retorna un perfil de voltages (alturas * canales) copiados desde el
463 data : retorna un perfil de voltages (alturas * canales) copiados desde el
460 buffer. Si no hay mas archivos a leer retorna None.
464 buffer. Si no hay mas archivos a leer retorna None.
461
465
462 Affected:
466 Affected:
463 self.dataOut
467 self.dataOut
464 self.profileIndex
468 self.profileIndex
465 self.flagDiscontinuousBlock
469 self.flagDiscontinuousBlock
466 self.flagIsNewBlock
470 self.flagIsNewBlock
467 '''
471 '''
468
472
469 err_counter = 0
473 err_counter = 0
470 self.dataOut.flagNoData = True
474 self.dataOut.flagNoData = True
471
475
472 if self.__isBufferEmpty():
476 if self.__isBufferEmpty():
473 self.__flagDiscontinuousBlock = False
477 self.__flagDiscontinuousBlock = False
474
478
475 while True:
479 while True:
476 if self.__readNextBlock():
480 if self.__readNextBlock():
477 break
481 break
478 if self.__thisUnixSample > self.__endUTCSecond*self.__sample_rate:
482 if self.__thisUnixSample > self.__endUTCSecond*self.__sample_rate:
479 return False
483 return False
480
484
481 if self.__flagDiscontinuousBlock:
485 if self.__flagDiscontinuousBlock:
482 print '[Reading] discontinuous block found ... continue with the next block'
486 print '[Reading] discontinuous block found ... continue with the next block'
483 continue
487 continue
484
488
485 if not self.__online:
489 if not self.__online:
486 return False
490 return False
487
491
488 err_counter += 1
492 err_counter += 1
489 if err_counter > nTries:
493 if err_counter > nTries:
490 return False
494 return False
491
495
492 print '[Reading] waiting %d seconds to read a new block' %seconds
496 print '[Reading] waiting %d seconds to read a new block' %seconds
493 sleep(seconds)
497 sleep(seconds)
494
498
495 self.dataOut.data = self.__data_buffer[:,self.__bufferIndex:self.__bufferIndex+self.__nSamples]
499 self.dataOut.data = self.__data_buffer[:,self.__bufferIndex:self.__bufferIndex+self.__nSamples]
496 self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex)/self.__sample_rate
500 self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex)/self.__sample_rate
497 self.dataOut.flagNoData = False
501 self.dataOut.flagNoData = False
498 self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
502 self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
499 self.dataOut.profileIndex = self.profileIndex
503 self.dataOut.profileIndex = self.profileIndex
500
504
501 self.__bufferIndex += self.__nSamples
505 self.__bufferIndex += self.__nSamples
502 self.profileIndex += 1
506 self.profileIndex += 1
503
507
504 if self.profileIndex == self.dataOut.nProfiles:
508 if self.profileIndex == self.dataOut.nProfiles:
505 self.profileIndex = 0
509 self.profileIndex = 0
506
510
507 return True
511 return True
508
512
509 def printInfo(self):
513 def printInfo(self):
510 '''
514 '''
511 '''
515 '''
512 if self.__printInfo == False:
516 if self.__printInfo == False:
513 return
517 return
514
518
515 # self.systemHeaderObj.printInfo()
519 # self.systemHeaderObj.printInfo()
516 # self.radarControllerHeaderObj.printInfo()
520 # self.radarControllerHeaderObj.printInfo()
517
521
518 self.__printInfo = False
522 self.__printInfo = False
519
523
520 def printNumberOfBlock(self):
524 def printNumberOfBlock(self):
521 '''
525 '''
522 '''
526 '''
523 return
527 return
524 # print self.profileIndex
528 # print self.profileIndex
525
529
526 ##@profile
530 ##@profile
527 def run(self, **kwargs):
531 def run(self, **kwargs):
528 '''
532 '''
529 This method will be called many times so here you should put all your code
533 This method will be called many times so here you should put all your code
530 '''
534 '''
531
535
532 if not self.isConfig:
536 if not self.isConfig:
533 self.setup(**kwargs)
537 self.setup(**kwargs)
534 print self.dataOut.dtype
535 self.i = self.i+1
538 self.i = self.i+1
536 self.getData(seconds=self.__delay)
539 self.getData(seconds=self.__delay)
537
540
538 return
541 return
539
542
540 class DigitalRFWriter(Operation):
543 class DigitalRFWriter(Operation):
541 '''
544 '''
542 classdocs
545 classdocs
543 '''
546 '''
544
547
545 def __init__(self, **kwargs):
548 def __init__(self, **kwargs):
546 '''
549 '''
547 Constructor
550 Constructor
548 '''
551 '''
549 Operation.__init__(self, **kwargs)
552 Operation.__init__(self, **kwargs)
550 self.metadata_dict = {}
553 self.metadata_dict = {}
551 self.dataOut = None
554 self.dataOut = None
552
555
556 def setHeader(self, dataOut):
557 return
558
553 def setup(self, dataOut, path, frequency, set=0, metadataFile='metadata', ext='.h5'):
559 def setup(self, dataOut, path, frequency, set=0, metadataFile='metadata', ext='.h5'):
554 '''
560 '''
555 In this method we should set all initial parameters.
561 In this method we should set all initial parameters.
556 Input:
562 Input:
557 dataOut: Input data will also be outputa data
563 dataOut: Input data will also be outputa data
558 '''
564 '''
559 self.metadata_dict['frequency'] = frequency
565 self.metadata_dict['frequency'] = dataOut.frequency
560 self.metadata_dict['timezone'] = -5 * 60 * 60
566 self.metadata_dict['timezone'] = dataOut.timeZone
561 self.metadata_dict['dtype'] = cPickle.dumps(dataOut.dtype)
567 self.metadata_dict['dtype'] = cPickle.dumps(dataOut.dtype)
568 self.metadata_dict['nProfiles'] = dataOut.nProfiles
569 self.metadata_dict['heightList'] = dataOut.heightList
570 self.metadata_dict['channelList'] = dataOut.channelList
571 self.metadata_dict['flagDecodeData'] = dataOut.flagDecodeData
572 self.metadata_dict['flagDeflipData'] = dataOut.flagDeflipData
573 self.metadata_dict['flagShiftFFT'] = dataOut.flagShiftFFT
574 self.metadata_dict['flagDataAsBlock'] = dataOut.flagDataAsBlock
575 self.metadata_dict['useLocalTime'] = dataOut.useLocalTime
576 self.metadata_dict['nCohInt'] = dataOut.nCohInt
562
577
563 self.__ippSeconds = dataOut.ippSeconds
578 self.__ippSeconds = dataOut.ippSeconds
564 self.__deltaH = dataOut.getDeltaH()
579 self.__deltaH = dataOut.getDeltaH()
565 self.__sample_rate = 1e6*0.15/self.__deltaH
580 self.__sample_rate = 1e6*0.15/self.__deltaH
566 self.__dtype = dataOut.dtype
581 self.__dtype = dataOut.dtype
567 if len(dataOut.dtype) == 2:
582 if len(dataOut.dtype) == 2:
568 self.__dtype = dataOut.dtype[0]
583 self.__dtype = dataOut.dtype[0]
569 self.__nSamples = dataOut.systemHeaderObj.nSamples
584 self.__nSamples = dataOut.systemHeaderObj.nSamples
570 self.__nProfiles = dataOut.nProfiles
585 self.__nProfiles = dataOut.nProfiles
571 self.__blocks_per_file = dataOut.processingHeaderObj.dataBlocksPerFile
586 self.__blocks_per_file = dataOut.processingHeaderObj.dataBlocksPerFile
572 self.arr_data = arr_data = numpy.ones((self.__nSamples, 1), dtype=[('r', self.__dtype), ('i', self.__dtype)])
587 self.arr_data = arr_data = numpy.ones((self.__nSamples, 1), dtype=[('r', self.__dtype), ('i', self.__dtype)])
573
588
574 file_cadence_millisecs = long(1.0 * self.__blocks_per_file * self.__nProfiles * self.__nSamples / self.__sample_rate) * 1000
589 file_cadence_millisecs = long(1.0 * self.__blocks_per_file * self.__nProfiles * self.__nSamples / self.__sample_rate) * 1000
575 sub_cadence_secs = file_cadence_millisecs / 500
590 sub_cadence_secs = file_cadence_millisecs / 500
576
591
577 print file_cadence_millisecs
592 print file_cadence_millisecs
578 print sub_cadence_secs
593 print sub_cadence_secs
579
594
580 sample_rate_fraction = Fraction(self.__sample_rate).limit_denominator()
595 sample_rate_fraction = Fraction(self.__sample_rate).limit_denominator()
581 sample_rate_numerator = long(sample_rate_fraction.numerator)
596 sample_rate_numerator = long(sample_rate_fraction.numerator)
582 sample_rate_denominator = long(sample_rate_fraction.denominator)
597 sample_rate_denominator = long(sample_rate_fraction.denominator)
583 start_global_index = dataOut.utctime * self.__sample_rate
598 start_global_index = dataOut.utctime * self.__sample_rate
584
599
585 uuid = 'prueba'
600 uuid = 'prueba'
586 compression_level = 1
601 compression_level = 1
587 checksum = False
602 checksum = False
588 is_complex = True
603 is_complex = True
589 num_subchannels = 1
604 num_subchannels = 1
590 is_continuous = True
605 is_continuous = True
591 marching_periods = False
606 marching_periods = False
592
607
593 self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, 100,
608 self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, 100,
594 1000, start_global_index,
609 1000, start_global_index,
595 sample_rate_numerator, sample_rate_denominator, uuid, compression_level, checksum,
610 sample_rate_numerator, sample_rate_denominator, uuid, compression_level, checksum,
596 is_complex, num_subchannels, is_continuous, marching_periods)
611 is_complex, num_subchannels, is_continuous, marching_periods)
597
612
598 metadata_dir = os.path.join(path, 'metadata')
613 metadata_dir = os.path.join(path, 'metadata')
599 os.system('mkdir %s' % (metadata_dir))
614 os.system('mkdir %s' % (metadata_dir))
600
615
601 self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, 100, 1, ##236, file_cadence_millisecs / 1000
616 self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, 100, 1, ##236, file_cadence_millisecs / 1000
602 sample_rate_numerator, sample_rate_denominator,
617 sample_rate_numerator, sample_rate_denominator,
603 metadataFile)
618 metadataFile)
604
619
605
620
606 self.isConfig = True
621 self.isConfig = True
607 self.currentSample = 0
622 self.currentSample = 0
608 return
623 return
609
624
610 def writeMetadata(self):
625 def writeMetadata(self):
611 print '[Writing] - Writing metadata'
626 print '[Writing] - Writing metadata'
612 start_idx = self.__sample_rate * self.dataOut.utctime
627 start_idx = self.__sample_rate * self.dataOut.utctime
613
628
614 self.metadata_dict['processingHeader'] = self.dataOut.processingHeaderObj.getAsDict()
629 self.metadata_dict['processingHeader'] = self.dataOut.processingHeaderObj.getAsDict()
615 self.metadata_dict['radarControllerHeader'] = self.dataOut.radarControllerHeaderObj.getAsDict()
630 self.metadata_dict['radarControllerHeader'] = self.dataOut.radarControllerHeaderObj.getAsDict()
616 self.metadata_dict['systemHeader'] = self.dataOut.systemHeaderObj.getAsDict()
631 self.metadata_dict['systemHeader'] = self.dataOut.systemHeaderObj.getAsDict()
617 self.digitalMetadataWriteObj.write(start_idx, self.metadata_dict)
632 self.digitalMetadataWriteObj.write(start_idx, self.metadata_dict)
618 return
633 return
619
634
620
635
621 def writeData(self):
636 def writeData(self):
622 for i in range(self.dataOut.systemHeaderObj.nSamples):
637 for i in range(self.dataOut.systemHeaderObj.nSamples):
623 self.arr_data[i]['r'] = self.dataOut.data[0][i].real
638 self.arr_data[i]['r'] = self.dataOut.data[0][i].real
624 self.arr_data[i]['i'] = self.dataOut.data[0][i].imag
639 self.arr_data[i]['i'] = self.dataOut.data[0][i].imag
625 self.digitalWriteObj.rf_write(self.arr_data)
640 self.digitalWriteObj.rf_write(self.arr_data)
626 return
641 return
627
642
628 def run(self, dataOut, frequency=49.92e6, path=None, **kwargs):
643 def run(self, dataOut, frequency=49.92e6, path=None, **kwargs):
629 '''
644 '''
630 This method will be called many times so here you should put all your code
645 This method will be called many times so here you should put all your code
631 Inputs:
646 Inputs:
632 dataOut: object with the data
647 dataOut: object with the data
633 '''
648 '''
634 # print dataOut.__dict__
649 # print dataOut.__dict__
635 self.dataOut = dataOut
650 self.dataOut = dataOut
636 if not self.isConfig:
651 if not self.isConfig:
637 self.setup(dataOut, path, frequency, **kwargs)
652 self.setup(dataOut, path, frequency, **kwargs)
638
653
639 self.writeData()
654 self.writeData()
640
655
641 self.currentSample += 1
656 self.currentSample += 1
642 if self.dataOut.flagDataAsBlock or self.currentSample == 1:
657 if self.dataOut.flagDataAsBlock or self.currentSample == 1:
643 self.writeMetadata()
658 self.writeMetadata()
644 if self.currentSample == self.__nProfiles: self.currentSample = 0
659 if self.currentSample == self.__nProfiles: self.currentSample = 0
645
660
646 def close(self):
661 def close(self):
647 print '[Writing] - Closing files '
662 print '[Writing] - Closing files '
648 try:
663 try:
649 self.digitalWriteObj.close()
664 self.digitalWriteObj.close()
650 except:
665 except:
651 pass
666 pass
652
667
653 # raise
668 # raise
654 if __name__ == '__main__':
669 if __name__ == '__main__':
655
670
656 readObj = DigitalRFReader()
671 readObj = DigitalRFReader()
657
672
658 while True:
673 while True:
659 readObj.run(path='/home/jchavez/jicamarca/mocked_data/')
674 readObj.run(path='/home/jchavez/jicamarca/mocked_data/')
660 # readObj.printInfo()
675 # readObj.printInfo()
661 # readObj.printNumberOfBlock()
676 # readObj.printNumberOfBlock()
General Comments 0
You need to be logged in to leave comments. Login now