##// END OF EJS Templates
schain
RSS
USRP Reade updated to read Sandra's data
Miguel Valdez -
r812:e72772d52a99
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,7 +1,7
1 1 '''
2 2 Created on Feb 7, 2012
3 3
4 4 @author $Author$
5 5 @version $Id$
6 6 '''
7 __version__ = "2.2.4" No newline at end of file
7 __version__ = "2.2.4.1" No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,587 +1,595
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import datetime
8 8 import numpy
9 9
10 10 try:
11 11 from gevent import sleep
12 12 except:
13 13 from time import sleep
14 14
15 15 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
16 16 from schainpy.model.data.jrodata import Voltage
17 17 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
18 18
19 19 try:
20 20 import digital_rf_hdf5
21 21 except:
22 22 print 'You should install "digital_rf_hdf5" module if you want to read USRP data'
23 23
24 24 class USRPReader(ProcessingUnit):
25 25 '''
26 26 classdocs
27 27 '''
28 28
29 29 def __init__(self):
30 30 '''
31 31 Constructor
32 32 '''
33 33
34 34 ProcessingUnit.__init__(self)
35 35
36 36 self.dataOut = Voltage()
37 37 self.__printInfo = True
38 38 self.__flagDiscontinuousBlock = False
39 39 self.__bufferIndex = 9999999
40 40
41 41 self.__ippKm = None
42 42 self.__codeType = 0
43 43 self.__nCode = None
44 44 self.__nBaud = None
45 45 self.__code = None
46 46
47 47 def __getCurrentSecond(self):
48 48
49 49 return self.__thisUnixSample/self.__sample_rate
50 50
51 51 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
52 52
53 53 def __setFileHeader(self):
54 54 '''
55 55 In this method will be initialized every parameter of dataOut object (header, no data)
56 56 '''
57 57 nProfiles = self.__sample_rate #Number of profiles by second
58 58
59 59 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
60 60 txA=0,
61 61 txB=0,
62 62 nWindows=1,
63 63 nHeights=self.__nSamples,
64 64 firstHeight=self.__firstHeigth,
65 65 deltaHeight=self.__deltaHeigth,
66 66 codeType=self.__codeType,
67 67 nCode=self.__nCode, nBaud=self.__nBaud,
68 68 code = self.__code)
69 69
70 70 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
71 71 nProfiles=nProfiles,
72 72 nChannels=len(self.__channelList),
73 73 adcResolution=14)
74 74
75 75 self.dataOut.type = "Voltage"
76 76
77 77 self.dataOut.data = None
78 78
79 79 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
80 80
81 81 # self.dataOut.nChannels = 0
82 82
83 83 # self.dataOut.nHeights = 0
84 84
85 85 self.dataOut.nProfiles = nProfiles
86 86
87 87 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
88 88
89 89 self.dataOut.channelList = self.__channelList
90 90
91 91 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
92 92
93 93 # self.dataOut.channelIndexList = None
94 94
95 95 self.dataOut.flagNoData = True
96 96
97 97 #Set to TRUE if the data is discontinuous
98 98 self.dataOut.flagDiscontinuousBlock = False
99 99
100 100 self.dataOut.utctime = None
101 101
102 102 self.dataOut.timeZone = self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
103 103
104 104 self.dataOut.dstFlag = 0
105 105
106 106 self.dataOut.errorCount = 0
107 107
108 108 self.dataOut.nCohInt = 1
109 109
110 110 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
111 111
112 112 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
113 113
114 114 self.dataOut.flagShiftFFT = False
115 115
116 116 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
117 117
118 118 #Time interval between profiles
119 119 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
120 120
121 121 self.dataOut.frequency = self.__frequency
122 122
123 123 self.dataOut.realtime = self.__online
124 124
125 125 def findDatafiles(self, path, startDate=None, endDate=None):
126 126
127 127 if not os.path.isdir(path):
128 128 return []
129 129
130 130 try:
131 131 digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
132 132 except:
133 133 digitalReadObj = digital_rf_hdf5.read_hdf5(path)
134 134
135 135 channelNameList = digitalReadObj.get_channels()
136 136
137 137 if not channelNameList:
138 138 return []
139 139
140 140 metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
141 141
142 142 sample_rate = metadata_dict['sample_rate'][0]
143 143
144 144 this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
145 145
146 146 try:
147 147 timezone = this_metadata_file['timezone'].value
148 148 except:
149 149 timezone = 0
150 150
151 151 startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(channelNameList[0])/sample_rate - timezone
152 152
153 153 startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
154 154 endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
155 155
156 156 if not startDate:
157 157 startDate = startDatetime.date()
158 158
159 159 if not endDate:
160 160 endDate = endDatatime.date()
161 161
162 162 dateList = []
163 163
164 164 thisDatetime = startDatetime
165 165
166 166 while(thisDatetime<=endDatatime):
167 167
168 168 thisDate = thisDatetime.date()
169 169
170 170 if thisDate < startDate:
171 171 continue
172 172
173 173 if thisDate > endDate:
174 174 break
175 175
176 176 dateList.append(thisDate)
177 177 thisDatetime += datetime.timedelta(1)
178 178
179 179 return dateList
180 180
181 181 def setup(self, path = None,
182 182 startDate = None,
183 183 endDate = None,
184 184 startTime = datetime.time(0,0,0),
185 185 endTime = datetime.time(23,59,59),
186 186 channelList = None,
187 187 nSamples = None,
188 188 ippKm = 60,
189 189 online = False,
190 190 delay = 60,
191 191 buffer_size = 1024,
192 192 **kwargs):
193 193 '''
194 194 In this method we should set all initial parameters.
195 195
196 196 Inputs:
197 197 path
198 198 startDate
199 199 endDate
200 200 startTime
201 201 endTime
202 202 set
203 203 expLabel
204 204 ext
205 205 online
206 206 delay
207 207 '''
208 208
209 209 if not os.path.isdir(path):
210 210 raise ValueError, "[Reading] Directory %s does not exist" %path
211 211
212 212 try:
213 213 self.digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
214 214 except:
215 215 self.digitalReadObj = digital_rf_hdf5.read_hdf5(path)
216 216
217 217 channelNameList = self.digitalReadObj.get_channels()
218 218
219 219 if not channelNameList:
220 220 raise ValueError, "[Reading] Directory %s does not have any files" %path
221 221
222 222 if not channelList:
223 223 channelList = range(len(channelNameList))
224 224
225 225 ########## Reading metadata ######################
226 226
227 227 metadata_dict = self.digitalReadObj.get_rf_file_metadata(channelNameList[channelList[0]])
228 228
229 229 self.__sample_rate = metadata_dict['sample_rate'][0]
230 self.__samples_per_file = metadata_dict['samples_per_file'][0]
230 # self.__samples_per_file = metadata_dict['samples_per_file'][0]
231 231 self.__deltaHeigth = 1e6*0.15/self.__sample_rate
232 232
233 233 this_metadata_file = self.digitalReadObj.get_metadata(channelNameList[channelList[0]])
234 234
235 self.__frequency = this_metadata_file['center_frequencies'].value
235 self.__frequency = None
236 try:
237 self.__frequency = this_metadata_file['center_frequencies'].value
238 except:
239 self.__frequency = this_metadata_file['fc'].value
240
241 if not self.__frequency:
242 raise ValueError, "Center Frequency is not defined in metadata file"
243
236 244 try:
237 245 self.__timezone = this_metadata_file['timezone'].value
238 246 except:
239 247 self.__timezone = 0
240 248
241 249 self.__firstHeigth = 0
242 250
243 251 try:
244 252 codeType = this_metadata_file['codeType'].value
245 253 except:
246 254 codeType = 0
247 255
248 256 nCode = 1
249 257 nBaud = 1
250 258 code = numpy.ones((nCode, nBaud), dtype=numpy.int)
251 259
252 260 if codeType:
253 261 nCode = this_metadata_file['nCode'].value
254 262 nBaud = this_metadata_file['nBaud'].value
255 263 code = this_metadata_file['code'].value
256 264
257 265 if not ippKm:
258 266 try:
259 267 #seconds to km
260 268 ippKm = 1e6*0.15*this_metadata_file['ipp'].value
261 269 except:
262 270 ippKm = None
263 271
264 272 ####################################################
265 273 startUTCSecond = None
266 274 endUTCSecond = None
267 275
268 276 if startDate:
269 277 startDatetime = datetime.datetime.combine(startDate, startTime)
270 278 startUTCSecond = (startDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
271 279
272 280 if endDate:
273 281 endDatetime = datetime.datetime.combine(endDate, endTime)
274 282 endUTCSecond = (endDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
275 283
276 284 start_index, end_index = self.digitalReadObj.get_bounds(channelNameList[channelList[0]])
277 285
278 286 if not startUTCSecond:
279 287 startUTCSecond = start_index/self.__sample_rate
280 288
281 289 if start_index > startUTCSecond*self.__sample_rate:
282 290 startUTCSecond = start_index/self.__sample_rate
283 291
284 292 if not endUTCSecond:
285 293 endUTCSecond = end_index/self.__sample_rate
286 294
287 295 if end_index < endUTCSecond*self.__sample_rate:
288 296 endUTCSecond = end_index/self.__sample_rate
289 297
290 298 if not nSamples:
291 299 if not ippKm:
292 300 raise ValueError, "[Reading] nSamples or ippKm should be defined"
293 301
294 302 nSamples = int(ippKm / (1e6*0.15/self.__sample_rate))
295 303
296 304 channelBoundList = []
297 305 channelNameListFiltered = []
298 306
299 307 for thisIndexChannel in channelList:
300 308 thisChannelName = channelNameList[thisIndexChannel]
301 309 start_index, end_index = self.digitalReadObj.get_bounds(thisChannelName)
302 310 channelBoundList.append((start_index, end_index))
303 311 channelNameListFiltered.append(thisChannelName)
304 312
305 313 self.profileIndex = 0
306 314
307 315 self.__delay = delay
308 316 self.__ippKm = ippKm
309 317 self.__codeType = codeType
310 318 self.__nCode = nCode
311 319 self.__nBaud = nBaud
312 320 self.__code = code
313 321
314 322 self.__datapath = path
315 323 self.__online = online
316 324 self.__channelList = channelList
317 325 self.__channelNameList = channelNameListFiltered
318 326 self.__channelBoundList = channelBoundList
319 327 self.__nSamples = nSamples
320 328 self.__samples_to_read = int(buffer_size*nSamples)
321 329 self.__nChannels = len(self.__channelList)
322 330
323 331 self.__startUTCSecond = startUTCSecond
324 332 self.__endUTCSecond = endUTCSecond
325 333
326 334 self.__timeInterval = 1.0 * self.__samples_to_read/self.__sample_rate #Time interval
327 335
328 336 if online:
329 337 # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
330 338 startUTCSecond = numpy.floor(endUTCSecond)
331 339
332 340 self.__thisUnixSample = int(startUTCSecond*self.__sample_rate) - self.__samples_to_read
333 341
334 342 self.__data_buffer = numpy.zeros((self.__nChannels, self.__samples_to_read), dtype = numpy.complex)
335 343
336 344 self.__setFileHeader()
337 345 self.isConfig = True
338 346
339 347 print "[Reading] USRP Data was found from %s to %s " %(
340 348 datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
341 349 datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
342 350 )
343 351
344 352 print "[Reading] Starting process from %s to %s" %(datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
345 353 datetime.datetime.utcfromtimestamp(endUTCSecond - self.__timezone)
346 354 )
347 355
348 356 def __reload(self):
349 357
350 358 if not self.__online:
351 359 return
352 360
353 361 # print
354 362 # print "%s not in range [%s, %s]" %(
355 363 # datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
356 364 # datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
357 365 # datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
358 366 # )
359 367 print "[Reading] reloading metadata ..."
360 368
361 369 try:
362 370 self.digitalReadObj.reload(complete_update=True)
363 371 except:
364 372 self.digitalReadObj.reload()
365 373
366 374 start_index, end_index = self.digitalReadObj.get_bounds(self.__channelNameList[self.__channelList[0]])
367 375
368 376 if start_index > self.__startUTCSecond*self.__sample_rate:
369 377 self.__startUTCSecond = 1.0*start_index/self.__sample_rate
370 378
371 379 if end_index > self.__endUTCSecond*self.__sample_rate:
372 380 self.__endUTCSecond = 1.0*end_index/self.__sample_rate
373 381 print
374 382 print "[Reading] New timerange found [%s, %s] " %(
375 383 datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
376 384 datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
377 385 )
378 386
379 387 return True
380 388
381 389 return False
382 390
383 391 def __readNextBlock(self, seconds=30, volt_scale = 218776):
384 392 '''
385 393 '''
386 394
387 395 #Set the next data
388 396 self.__flagDiscontinuousBlock = False
389 397 self.__thisUnixSample += self.__samples_to_read
390 398
391 399 if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
392 400 print "[Reading] There are no more data into selected time-range"
393 401
394 402 self.__reload()
395 403
396 404 if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
397 405 self.__thisUnixSample -= self.__samples_to_read
398 406 return False
399 407
400 408 indexChannel = 0
401 409
402 410 dataOk = False
403 411
404 412 for thisChannelName in self.__channelNameList:
405 413
406 414 try:
407 415 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
408 416 self.__samples_to_read,
409 417 thisChannelName)
410 418
411 419 except IOError, e:
412 420 #read next profile
413 421 self.__flagDiscontinuousBlock = True
414 422 print "[Reading] %s" %datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e
415 423 break
416 424
417 425 if result.shape[0] != self.__samples_to_read:
418 426 self.__flagDiscontinuousBlock = True
419 427 print "[Reading] %s: Too few samples were found, just %d/%d samples" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
420 428 result.shape[0],
421 429 self.__samples_to_read)
422 430 break
423 431
424 432 self.__data_buffer[indexChannel,:] = result*volt_scale
425 433
426 434 indexChannel += 1
427 435
428 436 dataOk = True
429 437
430 438 self.__utctime = self.__thisUnixSample/self.__sample_rate
431 439
432 440 if not dataOk:
433 441 return False
434 442
435 443 print "[Reading] %s: %d samples <> %f sec" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
436 444 self.__samples_to_read,
437 445 self.__timeInterval)
438 446
439 447 self.__bufferIndex = 0
440 448
441 449 return True
442 450
443 451 def __isBufferEmpty(self):
444 452
445 453 if self.__bufferIndex <= self.__samples_to_read - self.__nSamples:
446 454 return False
447 455
448 456 return True
449 457
450 458 def getData(self, seconds=30, nTries=5):
451 459
452 460 '''
453 461 This method gets the data from files and put the data into the dataOut object
454 462
455 463 In addition, increase el the buffer counter in one.
456 464
457 465 Return:
458 466 data : retorna un perfil de voltages (alturas * canales) copiados desde el
459 467 buffer. Si no hay mas archivos a leer retorna None.
460 468
461 469 Affected:
462 470 self.dataOut
463 471 self.profileIndex
464 472 self.flagDiscontinuousBlock
465 473 self.flagIsNewBlock
466 474 '''
467 475
468 476 err_counter = 0
469 477 self.dataOut.flagNoData = True
470 478
471 479 if self.__isBufferEmpty():
472 480
473 481 self.__flagDiscontinuousBlock = False
474 482
475 483 while True:
476 484 if self.__readNextBlock():
477 485 break
478 486
479 487 if self.__thisUnixSample > self.__endUTCSecond*self.__sample_rate:
480 488 return False
481 489
482 490 if self.__flagDiscontinuousBlock:
483 491 print '[Reading] discontinuous block found ... continue with the next block'
484 492 continue
485 493
486 494 if not self.__online:
487 495 return False
488 496
489 497 err_counter += 1
490 498 if err_counter > nTries:
491 499 return False
492 500
493 501 print '[Reading] waiting %d seconds to read a new block' %seconds
494 502 sleep(seconds)
495 503
496 504 self.dataOut.data = self.__data_buffer[:,self.__bufferIndex:self.__bufferIndex+self.__nSamples]
497 505 self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex)/self.__sample_rate
498 506 self.dataOut.flagNoData = False
499 507 self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
500 508
501 509 self.__bufferIndex += self.__nSamples
502 510 self.profileIndex += 1
503 511
504 512 return True
505 513
506 514 def printInfo(self):
507 515 '''
508 516 '''
509 517 if self.__printInfo == False:
510 518 return
511 519
512 520 # self.systemHeaderObj.printInfo()
513 521 # self.radarControllerHeaderObj.printInfo()
514 522
515 523 self.__printInfo = False
516 524
517 525 def printNumberOfBlock(self):
518 526 '''
519 527 '''
520 528
521 529 print self.profileIndex
522 530
523 531 def run(self, **kwargs):
524 532 '''
525 533 This method will be called many times so here you should put all your code
526 534 '''
527 535
528 536 if not self.isConfig:
529 537 self.setup(**kwargs)
530 538
531 539 self.getData(seconds=self.__delay)
532 540
533 541 return
534 542
535 543 class USRPWriter(Operation):
536 544 '''
537 545 classdocs
538 546 '''
539 547
540 548 def __init__(self):
541 549 '''
542 550 Constructor
543 551 '''
544 552 self.dataOut = None
545 553
546 554 def setup(self, dataIn, path, blocksPerFile, set=0, ext=None):
547 555 '''
548 556 In this method we should set all initial parameters.
549 557
550 558 Input:
551 559 dataIn : Input data will also be outputa data
552 560
553 561 '''
554 562 self.dataOut = dataIn
555 563
556 564
557 565
558 566
559 567
560 568 self.isConfig = True
561 569
562 570 return
563 571
564 572 def run(self, dataIn, **kwargs):
565 573 '''
566 574 This method will be called many times so here you should put all your code
567 575
568 576 Inputs:
569 577
570 578 dataIn : object with the data
571 579
572 580 '''
573 581
574 582 if not self.isConfig:
575 583 self.setup(dataIn, **kwargs)
576 584
577 585
578 586 if __name__ == '__main__':
579 587
580 588 readObj = USRPReader()
581 589
582 590 while True:
583 591 readObj.run(path='/Volumes/DATA/haystack/passive_radar/')
584 592 # readObj.printInfo()
585 593 readObj.printNumberOfBlock()
586 594
587 595 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now