##// END OF EJS Templates
schain
RSS
tests
Jose Chavez -
r991:584188041a2c
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,15
1 Lectura
2 200samples -> 0.47m HDD 1000''
3 200samples -> 6 HDD 100ms file 100s folder
4 200samples -> 0.48ms HDD 100ms file 1000s folder
5 200samples -> 116ms HDD 5000ms file 100s folder
6 200samples -> 182 HDD 10000ms file 100s folder
7 200samples -> 143 HDD 10000ms file 100s folder SSD
8
9 Escritura
10 200samples -> 0.78m HDD 100ms file 1000s folder
11 200samples -> 0.066m HDD 100ms file 1000s folder
12 200samples -> 0.30 HDD 100ms file 100s folder
13 200samples -> 0.23 HDD 5000ms file 100s folder
14 200samples -> 0.176 HDD 10000ms file 100s folder
15
Show file before | Show file after | Hide comments
@@ -1,697 +1,730
1 1
2 2 '''
3 3 Created on Jul 3, 2014
4 4
5 5 @author: roj-idl71
6 6 '''
7 # SUBCHANNELS EN VEZ DE CHANNELS
8 # BENCHMARKS -> PROBLEMAS CON ARCHIVOS GRANDES -> INCONSTANTE EN EL TIEMPO
9 # ACTUALIZACION DE VERSION
10 # HEADERS
11 # MODULO DE ESCRITURA
12 # METADATA
13
7 14 import os
8 15 import datetime
9 16 import numpy
10 17 import timeit
11 18 from profilehooks import coverage, profile
12 19 from fractions import Fraction
13 20
14 21 try:
15 22 from gevent import sleep
16 23 except:
17 24 from time import sleep
18 25
19 26 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
20 27 from schainpy.model.data.jrodata import Voltage
21 28 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
22 29 from time import time
23 30
24 31 import cPickle
25 32 try:
26 33 import digital_rf
27 34 except:
28 35 print 'You should install "digital_rf" module if you want to read Digital RF data'
29 36
30 37 class DigitalRFReader(ProcessingUnit):
31 38 '''
32 39 classdocs
33 40 '''
34 41
35 42 def __init__(self, **kwargs):
36 43 '''
37 44 Constructor
38 45 '''
39 46
40 47 ProcessingUnit.__init__(self, **kwargs)
41 48
42 49 self.dataOut = Voltage()
43 50 self.__printInfo = True
44 51 self.__flagDiscontinuousBlock = False
45 52 self.__bufferIndex = 9999999
46 53 self.__ippKm = None
47 54 self.__codeType = 0
48 55 self.__nCode = None
49 56 self.__nBaud = None
50 57 self.__code = None
51 58
59 def close(self):
60 print 'Average of writing to digital rf format is ', self.oldAverage * 1000
61 return
62
52 63 def __getCurrentSecond(self):
53 64
54 65 return self.__thisUnixSample/self.__sample_rate
55 66
56 67 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
57 68
58 69 def __setFileHeader(self):
59 70 '''
60 71 In this method will be initialized every parameter of dataOut object (header, no data)
61 72 '''
62 73 ippSeconds = 1.0*self.__nSamples/self.__sample_rate
63 74
64 75 nProfiles = 1.0/ippSeconds # Number of profiles in one second
65 76
66 77 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(self.__radarControllerHeader)
67 78
68 79 self.dataOut.systemHeaderObj = SystemHeader(self.__systemHeader)
69 80
70 81 self.dataOut.type = "Voltage"
71 82
72 83 self.dataOut.data = None
73 84
74 85 self.dataOut.dtype = self.dtype
75 86
76 87 # self.dataOut.nChannels = 0
77 88
78 89 # self.dataOut.nHeights = 0
79 90
80 91 self.dataOut.nProfiles = nProfiles
81 92
82 93 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
83 94
84 95 self.dataOut.channelList = range(self.__num_subchannels)
85 96
86 97 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
87 98
88 99 # self.dataOut.channelIndexList = None
89 100
90 101 self.dataOut.flagNoData = True
91 102
92 103 self.dataOut.flagDataAsBlock = False
93 104 # Set to TRUE if the data is discontinuous
94 105 self.dataOut.flagDiscontinuousBlock = False
95 106
96 107 self.dataOut.utctime = None
97 108
98 109 self.dataOut.timeZone = self.__timezone/60 # timezone like jroheader, difference in minutes between UTC and localtime
99 110
100 111 self.dataOut.dstFlag = 0
101 112
102 113 self.dataOut.errorCount = 0
103 114
104 115 self.dataOut.nCohInt = self.fixed_metadata_dict['nCohInt']
105 116
106 117 self.dataOut.flagDecodeData = self.fixed_metadata_dict['flagDecodeData'] # asumo que la data esta decodificada
107 118
108 119 self.dataOut.flagDeflipData = self.fixed_metadata_dict['flagDeflipData'] # asumo que la data esta sin flip
109 120
110 121 self.dataOut.flagShiftFFT = self.fixed_metadata_dict['flagShiftFFT']
111 122
112 123 self.dataOut.useLocalTime = self.fixed_metadata_dict['useLocalTime']
113 124
114 125 self.dataOut.ippSeconds = ippSeconds
115 126
116 127 # Time interval between profiles
117 128 # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
118 129
119 130 self.dataOut.frequency = self.__frequency
120 131
121 132 self.dataOut.realtime = self.__online
122 133
123 134 def findDatafiles(self, path, startDate=None, endDate=None):
124 135
125 136 if not os.path.isdir(path):
126 137 return []
127 138
128 139 try:
129 140 digitalReadObj = digital_rf.DigitalRFReader(path, load_all_metadata=True)
130 141 except:
131 142 digitalReadObj = digital_rf.DigitalRFReader(path)
132 143
133 144 channelNameList = digitalReadObj.get_channels()
134 145
135 146 if not channelNameList:
136 147 return []
137 148
138 149 metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
139 150
140 151 sample_rate = metadata_dict['sample_rate'][0]
141 152
142 153 this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
143 154
144 155 try:
145 156 timezone = this_metadata_file['timezone'].value
146 157 except:
147 158 timezone = 0
148 159
149 160 startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(channelNameList[0])/sample_rate - timezone
150 161
151 162 startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
152 163 endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
153 164
154 165 if not startDate:
155 166 startDate = startDatetime.date()
156 167
157 168 if not endDate:
158 169 endDate = endDatatime.date()
159 170
160 171 dateList = []
161 172
162 173 thisDatetime = startDatetime
163 174
164 175 while(thisDatetime<=endDatatime):
165 176
166 177 thisDate = thisDatetime.date()
167 178
168 179 if thisDate < startDate:
169 180 continue
170 181
171 182 if thisDate > endDate:
172 183 break
173 184
174 185 dateList.append(thisDate)
175 186 thisDatetime += datetime.timedelta(1)
176 187
177 188 return dateList
178 189
179 190 def setup(self, path = None,
180 191 startDate = None,
181 192 endDate = None,
182 193 startTime = datetime.time(0,0,0),
183 194 endTime = datetime.time(23,59,59),
184 195 channelList = None,
185 196 nSamples = None,
186 197 online = False,
187 198 delay = 60,
188 199 buffer_size = 1024,
189 200 ippKm=None,
190 201 **kwargs):
191 202 '''
192 203 In this method we should set all initial parameters.
193 204
194 205 Inputs:
195 206 path
196 207 startDate
197 208 endDate
198 209 startTime
199 210 endTime
200 211 set
201 212 expLabel
202 213 ext
203 214 online
204 215 delay
205 216 '''
206 217
207 218 if not os.path.isdir(path):
208 219 raise ValueError, "[Reading] Directory %s does not exist" %path
209 220
210 221 try:
211 222 self.digitalReadObj = digital_rf.DigitalRFReader(path, load_all_metadata=True)
212 223 except:
213 224 self.digitalReadObj = digital_rf.DigitalRFReader(path)
214 225
215 226 channelNameList = self.digitalReadObj.get_channels()
216 227
217 228 if not channelNameList:
218 229 raise ValueError, "[Reading] Directory %s does not have any files" %path
219 230
220 231 if not channelList:
221 232 channelList = range(len(channelNameList))
222 233
223 234
224 235 ########## Reading metadata ######################
225 236
226 237 top_properties = self.digitalReadObj.get_properties(channelNameList[channelList[0]])
227 238
228 239
229 240
230 241
231 242
232 243
233 244 self.__num_subchannels = top_properties['num_subchannels']
234 245 self.__sample_rate = 1.0 * top_properties['sample_rate_numerator'] / top_properties['sample_rate_denominator']
235 246
236 247 # self.__samples_per_file = top_properties['samples_per_file'][0]
237 248 self.__deltaHeigth = 1e6*0.15/self.__sample_rate ## why 0.15?
238 249
239 250 this_metadata_file = self.digitalReadObj.get_digital_metadata(channelNameList[channelList[0]])
240 251 print this_metadata_file
241 252 metadata_bounds = this_metadata_file.get_bounds()
242 253 self.fixed_metadata_dict = this_metadata_file.read(metadata_bounds[0])[metadata_bounds[0]] ## GET FIRST HEADER
243 254 self.__processingHeader = self.fixed_metadata_dict['processingHeader']
244 255 self.__radarControllerHeader = self.fixed_metadata_dict['radarControllerHeader']
245 256 self.__systemHeader = self.fixed_metadata_dict['systemHeader']
246 257 self.dtype = cPickle.loads(self.fixed_metadata_dict['dtype'])
247 258
248 259 self.__frequency = None
249 260
250 261 try:
251 262 self.__frequency = self.fixed_metadata_dict['frequency']
252 263 except:
253 264 self.__frequency = None
254 265
255 266 try:
256 267 self.__timezone = self.fixed_metadata_dict['timezone'] * 60
257 268 except:
258 269 self.__timezone = 0
259 270
260 271
261 272 try:
262 273 nSamples = self.fixed_metadata_dict['nSamples']
263 274 except:
264 275 nSamples = None
265 276
266 277 self.__firstHeigth = 0
267 278
268 279 try:
269 280 codeType = self.__radarControllerHeader['codeType']
270 281 except:
271 282 codeType = 0
272 283
273 284 nCode = 1
274 285 nBaud = 1
275 286 code = numpy.ones((nCode, nBaud), dtype=numpy.int)
276 287
277 288 if codeType:
278 289 nCode = self.__radarControllerHeader['nCode']
279 290 nBaud = self.__radarControllerHeader['nBaud']
280 291 code = self.__radarControllerHeader['code']
281 292
282 293 if not ippKm:
283 294 try:
284 295 # seconds to km
285 296 ippKm = self.__radarControllerHeader['ipp']
286 297 except:
287 298 ippKm = None
288 299 ####################################################
289 300 startUTCSecond = None
290 301 endUTCSecond = None
291 302
292 303 if startDate:
293 304 startDatetime = datetime.datetime.combine(startDate, startTime)
294 305 startUTCSecond = (startDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
295 306
296 307 if endDate:
297 308 endDatetime = datetime.datetime.combine(endDate, endTime)
298 309 endUTCSecond = (endDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
299 310
300 311 start_index, end_index = self.digitalReadObj.get_bounds(channelNameList[channelList[0]])
301 312
302 313 if not startUTCSecond:
303 314 startUTCSecond = start_index/self.__sample_rate
304 315
305 316 if start_index > startUTCSecond*self.__sample_rate:
306 317 startUTCSecond = start_index/self.__sample_rate
307 318
308 319 if not endUTCSecond:
309 320 endUTCSecond = end_index/self.__sample_rate
310 321
311 322 if end_index < endUTCSecond*self.__sample_rate:
312 323 endUTCSecond = end_index/self.__sample_rate
313 324 print ippKm
314 325 if not nSamples:
315 326 if not ippKm:
316 327 raise ValueError, "[Reading] nSamples or ippKm should be defined"
317 328 nSamples = int(ippKm / (1e6*0.15/self.__sample_rate))
318 329 print nSamples
319 330 channelBoundList = []
320 331 channelNameListFiltered = []
321 332
322 333 for thisIndexChannel in channelList:
323 334 thisChannelName = channelNameList[thisIndexChannel]
324 335 start_index, end_index = self.digitalReadObj.get_bounds(thisChannelName)
325 336 channelBoundList.append((start_index, end_index))
326 337 channelNameListFiltered.append(thisChannelName)
327 338
328 339 self.profileIndex = 0
329 340 self.i= 0
330 341 self.__delay = delay
331 342 self.__ippKm = ippKm
332 343 self.__codeType = codeType
333 344 self.__nCode = nCode
334 345 self.__nBaud = nBaud
335 346 self.__code = code
336 347
337 348 self.__datapath = path
338 349 self.__online = online
339 350 self.__channelList = channelList
340 351 self.__channelNameList = channelNameListFiltered
341 352 self.__channelBoundList = channelBoundList
342 353 self.__nSamples = nSamples
343 354 self.__samples_to_read = long(nSamples) # FIJO: AHORA 40
344 355 self.__nChannels = len(self.__channelList)
345 356
346 357 self.__startUTCSecond = startUTCSecond
347 358 self.__endUTCSecond = endUTCSecond
348 359
349 360 self.__timeInterval = 1.0 * self.__samples_to_read/self.__sample_rate # Time interval
350 361
351 362 if online:
352 363 # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
353 364 startUTCSecond = numpy.floor(endUTCSecond)
354 365
355 366 self.__thisUnixSample = long(startUTCSecond*self.__sample_rate) - self.__samples_to_read ## por que en el otro metodo lo primero q se hace es sumar samplestoread
356 367
357 368 self.__data_buffer = numpy.zeros((self.__num_subchannels, self.__samples_to_read), dtype = numpy.complex)
358 369
359 370 self.__setFileHeader()
360 371 self.isConfig = True
361 372
362 373 print "[Reading] Digital RF Data was found from %s to %s " %(
363 374 datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
364 375 datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
365 376 )
366 377
367 378 print "[Reading] Starting process from %s to %s" %(datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
368 379 datetime.datetime.utcfromtimestamp(endUTCSecond - self.__timezone)
369 380 )
370
381 self.oldAverage = None
382 self.count = 0
383 self.executionTime = 0
371 384 def __reload(self):
372 385 # print
373 386 # print "%s not in range [%s, %s]" %(
374 387 # datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
375 388 # datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
376 389 # datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
377 390 # )
378 391 print "[Reading] reloading metadata ..."
379 392
380 393 try:
381 394 self.digitalReadObj.reload(complete_update=True)
382 395 except:
383 396 self.digitalReadObj.reload()
384 397
385 398 start_index, end_index = self.digitalReadObj.get_bounds(self.__channelNameList[self.__channelList[0]])
386 399
387 400 if start_index > self.__startUTCSecond*self.__sample_rate:
388 401 self.__startUTCSecond = 1.0*start_index/self.__sample_rate
389 402
390 403 if end_index > self.__endUTCSecond*self.__sample_rate:
391 404 self.__endUTCSecond = 1.0*end_index/self.__sample_rate
392 405 print
393 406 print "[Reading] New timerange found [%s, %s] " %(
394 407 datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
395 408 datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
396 409 )
397 410
398 411 return True
399 412
400 413 return False
401 414
415 def timeit(self, toExecute):
416 t0 = time()
417 toExecute()
418 self.executionTime = time() - t0
419 if self.oldAverage is None: self.oldAverage = self.executionTime
420 self.oldAverage = (self.executionTime + self.count*self.oldAverage) / (self.count + 1.0)
421 self.count = self.count + 1.0
422 return
423
402 424 def __readNextBlock(self, seconds=30, volt_scale = 1):
403 425 '''
404 426 '''
405 427
406 428 # Set the next data
407 429 self.__flagDiscontinuousBlock = False
408 430 self.__thisUnixSample += self.__samples_to_read
409 431
410 432 if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
411 433 print "[Reading] There are no more data into selected time-range"
412 434 if self.__online:
413 435 self.__reload()
414 436 else:
415 437 return False
416 438
417 439 if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
418 440 return False
419 441 self.__thisUnixSample -= self.__samples_to_read
420 442
421 443 indexChannel = 0
422 444
423 445 dataOk = False
424 446 for thisChannelName in self.__channelNameList: ##TODO VARIOS CHANNELS?
425 447 for indexSubchannel in range(self.__num_subchannels):
426 448 try:
449 t0 = time()
427 450 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
428 self.__samples_to_read,
429 thisChannelName, sub_channel=indexSubchannel)
451 self.__samples_to_read,
452 thisChannelName, sub_channel=indexSubchannel)
453 self.executionTime = time() - t0
454 if self.oldAverage is None: self.oldAverage = self.executionTime
455 self.oldAverage = (self.executionTime + self.count*self.oldAverage) / (self.count + 1.0)
456 self.count = self.count + 1.0
457
430 458 except IOError, e:
431 459 #read next profile
432 460 self.__flagDiscontinuousBlock = True
433 461 print "[Reading] %s" %datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e
434 462 break
435 463
436 464 if result.shape[0] != self.__samples_to_read:
437 465 self.__flagDiscontinuousBlock = True
438 466 print "[Reading] %s: Too few samples were found, just %d/%d samples" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
439 467 result.shape[0],
440 468 self.__samples_to_read)
441 469 break
442 470
443 471 self.__data_buffer[indexSubchannel,:] = result*volt_scale
444 472
445 473 indexChannel += 1
446 474
447 475 dataOk = True
448 476
449 477 self.__utctime = self.__thisUnixSample/self.__sample_rate
450 478
451 479 if not dataOk:
452 480 return False
453 481
454 482 print "[Reading] %s: %d samples <> %f sec" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
455 483 self.__samples_to_read,
456 484 self.__timeInterval)
457 485
458 486 self.__bufferIndex = 0
459 487
460 488 return True
461 489
462 490 def __isBufferEmpty(self):
463 491 return self.__bufferIndex > self.__samples_to_read - self.__nSamples #40960 - 40
464 492
465 493 def getData(self, seconds=30, nTries=5):
466 494
467 495 '''
468 496 This method gets the data from files and put the data into the dataOut object
469 497
470 498 In addition, increase el the buffer counter in one.
471 499
472 500 Return:
473 501 data : retorna un perfil de voltages (alturas * canales) copiados desde el
474 502 buffer. Si no hay mas archivos a leer retorna None.
475 503
476 504 Affected:
477 505 self.dataOut
478 506 self.profileIndex
479 507 self.flagDiscontinuousBlock
480 508 self.flagIsNewBlock
481 509 '''
482 510
483 511 err_counter = 0
484 512 self.dataOut.flagNoData = True
485 513
486 514 if self.__isBufferEmpty():
487 515 self.__flagDiscontinuousBlock = False
488 516
489 517 while True:
490 518 if self.__readNextBlock():
491 519 break
492 520 if self.__thisUnixSample > self.__endUTCSecond*self.__sample_rate:
493 521 return False
494 522
495 523 if self.__flagDiscontinuousBlock:
496 524 print '[Reading] discontinuous block found ... continue with the next block'
497 525 continue
498 526
499 527 if not self.__online:
500 528 return False
501 529
502 530 err_counter += 1
503 531 if err_counter > nTries:
504 532 return False
505 533
506 534 print '[Reading] waiting %d seconds to read a new block' %seconds
507 535 sleep(seconds)
508 536
509 537 self.dataOut.data = self.__data_buffer[:,self.__bufferIndex:self.__bufferIndex+self.__nSamples]
510 538 self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex)/self.__sample_rate
511 539 self.dataOut.flagNoData = False
512 540 self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
513 541 self.dataOut.profileIndex = self.profileIndex
514 542
515 543 self.__bufferIndex += self.__nSamples
516 544 self.profileIndex += 1
517 545
518 546 if self.profileIndex == self.dataOut.nProfiles:
519 547 self.profileIndex = 0
520 548
521 549 return True
522 550
523 551 def printInfo(self):
524 552 '''
525 553 '''
526 554 if self.__printInfo == False:
527 555 return
528 556
529 557 # self.systemHeaderObj.printInfo()
530 558 # self.radarControllerHeaderObj.printInfo()
531 559
532 560 self.__printInfo = False
533 561
534 562 def printNumberOfBlock(self):
535 563 '''
536 564 '''
537 565 return
538 566 # print self.profileIndex
539 567
540 568 ##@profile
541 569 def run(self, **kwargs):
542 570 '''
543 571 This method will be called many times so here you should put all your code
544 572 '''
545 573
546 574 if not self.isConfig:
547 575 self.setup(**kwargs)
548 576 self.i = self.i+1
549 577 self.getData(seconds=self.__delay)
550 578
551 579 return
552 580
553 581 class DigitalRFWriter(Operation):
554 582 '''
555 583 classdocs
556 584 '''
557 585
558 586 def __init__(self, **kwargs):
559 587 '''
560 588 Constructor
561 589 '''
562 590 Operation.__init__(self, **kwargs)
563 591 self.metadata_dict = {}
564 self.dataOut = None
565
566 def setHeader(self, dataOut):
592 self.dataOut = None
593
594 def setHeader(self):
595
596 self.metadata_dict['frequency'] = self.dataOut.frequency
597 self.metadata_dict['timezone'] = self.dataOut.timeZone
598 self.metadata_dict['dtype'] = cPickle.dumps(self.dataOut.dtype)
599 self.metadata_dict['nProfiles'] = self.dataOut.nProfiles
600 self.metadata_dict['heightList'] = self.dataOut.heightList
601 self.metadata_dict['channelList'] = self.dataOut.channelList
602 self.metadata_dict['flagDecodeData'] = self.dataOut.flagDecodeData
603 self.metadata_dict['flagDeflipData'] = self.dataOut.flagDeflipData
604 self.metadata_dict['flagShiftFFT'] = self.dataOut.flagShiftFFT
605 self.metadata_dict['flagDataAsBlock'] = self.dataOut.flagDataAsBlock
606 self.metadata_dict['useLocalTime'] = self.dataOut.useLocalTime
607 self.metadata_dict['nCohInt'] = self.dataOut.nCohInt
608
567 609 return
568 610
569 def setup(self, dataOut, path, frequency, set=0, metadataFile='metadata', ext='.h5'):
611 def setup(self, dataOut, path, frequency, fileCadence, dirCadence, metadataCadence, set=0, metadataFile='metadata', ext='.h5'):
570 612 '''
571 613 In this method we should set all initial parameters.
572 614 Input:
573 615 dataOut: Input data will also be outputa data
574 616 '''
575 self.metadata_dict['frequency'] = dataOut.frequency
576 self.metadata_dict['timezone'] = dataOut.timeZone
577 self.metadata_dict['dtype'] = cPickle.dumps(dataOut.dtype)
578 self.metadata_dict['nProfiles'] = dataOut.nProfiles
579 self.metadata_dict['heightList'] = dataOut.heightList
580 self.metadata_dict['channelList'] = dataOut.channelList
581 self.metadata_dict['flagDecodeData'] = dataOut.flagDecodeData
582 self.metadata_dict['flagDeflipData'] = dataOut.flagDeflipData
583 self.metadata_dict['flagShiftFFT'] = dataOut.flagShiftFFT
584 self.metadata_dict['flagDataAsBlock'] = dataOut.flagDataAsBlock
585 self.metadata_dict['useLocalTime'] = dataOut.useLocalTime
586 self.metadata_dict['nCohInt'] = dataOut.nCohInt
587
617 self.setHeader()
588 618 self.__ippSeconds = dataOut.ippSeconds
589 619 self.__deltaH = dataOut.getDeltaH()
590 620 self.__sample_rate = 1e6*0.15/self.__deltaH
591 621 self.__dtype = dataOut.dtype
592 622 if len(dataOut.dtype) == 2:
593 623 self.__dtype = dataOut.dtype[0]
594 624 self.__nSamples = dataOut.systemHeaderObj.nSamples
595 625 self.__nProfiles = dataOut.nProfiles
596 626 self.__blocks_per_file = dataOut.processingHeaderObj.dataBlocksPerFile
597 627 self.arr_data = arr_data = numpy.ones((self.__nSamples, 2), dtype=[('r', self.__dtype), ('i', self.__dtype)])
598 628
599 629 file_cadence_millisecs = long(1.0 * self.__blocks_per_file * self.__nProfiles * self.__nSamples / self.__sample_rate) * 1000
600 630 sub_cadence_secs = file_cadence_millisecs / 500
601 631
602 print file_cadence_millisecs
603 print sub_cadence_secs
604
605 632 sample_rate_fraction = Fraction(self.__sample_rate).limit_denominator()
606 633 sample_rate_numerator = long(sample_rate_fraction.numerator)
607 634 sample_rate_denominator = long(sample_rate_fraction.denominator)
608 635 start_global_index = dataOut.utctime * self.__sample_rate
609 636
610 637 uuid = 'prueba'
611 638 compression_level = 1
612 639 checksum = False
613 640 is_complex = True
614 641 num_subchannels = len(dataOut.channelList)
615 642 is_continuous = True
616 643 marching_periods = False
617 644
618 self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, 100,
619 100, start_global_index,
645 self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, dirCadence,
646 fileCadence, start_global_index,
620 647 sample_rate_numerator, sample_rate_denominator, uuid, compression_level, checksum,
621 648 is_complex, num_subchannels, is_continuous, marching_periods)
622 649
623 650 metadata_dir = os.path.join(path, 'metadata')
624 651 os.system('mkdir %s' % (metadata_dir))
625 652
626 self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, 100, 1, ##236, file_cadence_millisecs / 1000
653 self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, dirCadence, 1, ##236, file_cadence_millisecs / 1000
627 654 sample_rate_numerator, sample_rate_denominator,
628 655 metadataFile)
629 656
630 657
631 658 self.isConfig = True
632 659 self.currentSample = 0
633 660 self.oldAverage = 0
634 661 self.count = 0
635 662 return
636 663
637 664 def writeMetadata(self):
638 665 print '[Writing] - Writing metadata'
639 666 start_idx = self.__sample_rate * self.dataOut.utctime
640 667
641 668 self.metadata_dict['processingHeader'] = self.dataOut.processingHeaderObj.getAsDict()
642 669 self.metadata_dict['radarControllerHeader'] = self.dataOut.radarControllerHeaderObj.getAsDict()
643 670 self.metadata_dict['systemHeader'] = self.dataOut.systemHeaderObj.getAsDict()
644 671 self.digitalMetadataWriteObj.write(start_idx, self.metadata_dict)
645 672 return
646 673
647 674
648 def writeData(self):
649 for i in range(self.dataOut.systemHeaderObj.nSamples):
650 for channel in self.dataOut.channelList:
651 self.arr_data[i][channel]['r'] = self.dataOut.data[channel][i].real
652 self.arr_data[i][channel]['i'] = self.dataOut.data[channel][i].imag
675 def timeit(self, toExecute):
653 676 t0 = time()
654 self.digitalWriteObj.rf_write(self.arr_data)
677 toExecute()
655 678 self.executionTime = time() - t0
656 679 if self.oldAverage is None: self.oldAverage = self.executionTime
657 680 self.oldAverage = (self.executionTime + self.count*self.oldAverage) / (self.count + 1.0)
658 681 self.count = self.count + 1.0
659 if self.oldAverage is None: self.oldAverage = self.executionTime
682 return
683
684
685 def writeData(self):
686 for i in range(self.dataOut.systemHeaderObj.nSamples):
687 for channel in self.dataOut.channelList:
688 self.arr_data[i][channel]['r'] = self.dataOut.data[channel][i].real
689 self.arr_data[i][channel]['i'] = self.dataOut.data[channel][i].imag
690
691 def f(): return self.digitalWriteObj.rf_write(self.arr_data)
692 self.timeit(f)
660 693
661 694 return
662 695
663 def run(self, dataOut, frequency=49.92e6, path=None, **kwargs):
696 def run(self, dataOut, frequency=49.92e6, path=None, fileCadence=1000, dirCadence=100, metadataCadence=1, **kwargs):
664 697 '''
665 698 This method will be called many times so here you should put all your code
666 699 Inputs:
667 700 dataOut: object with the data
668 701 '''
669 702 # print dataOut.__dict__
670 703 self.dataOut = dataOut
671 704 if not self.isConfig:
672 self.setup(dataOut, path, frequency, **kwargs)
705 self.setup(dataOut, path, frequency, fileCadence, dirCadence, metadataCadence, **kwargs)
673 706
674 707 self.writeData()
675 708
676 709 self.currentSample += 1
677 710 if self.dataOut.flagDataAsBlock or self.currentSample == 1:
678 711 self.writeMetadata()
679 712 if self.currentSample == self.__nProfiles: self.currentSample = 0
680 713
681 714 def close(self):
682 715 print '[Writing] - Closing files '
683 716 print 'Average of writing to digital rf format is ', self.oldAverage * 1000
684 717 try:
685 718 self.digitalWriteObj.close()
686 719 except:
687 720 pass
688 721
689 722 # raise
690 723 if __name__ == '__main__':
691 724
692 725 readObj = DigitalRFReader()
693 726
694 727 while True:
695 728 readObj.run(path='/home/jchavez/jicamarca/mocked_data/')
696 729 # readObj.printInfo()
697 730 # readObj.printNumberOfBlock()
Show file before | Show file after | Hide comments
@@ -1,1 +1,1
1 <Project description="Testing USRP data reader" id="191" name="test01"><ReadUnit datatype="DigitalRF" id="1911" inputId="0" name="DigitalRFReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="DigitalRF" /><Parameter format="str" id="191112" name="path" value="/home/jchavez/jicamarca/mocked_data" /><Parameter format="date" id="191113" name="startDate" value="2000/07/03" /><Parameter format="date" id="191114" name="endDate" value="2017/07/03" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="Voltage" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="Scope" priority="2" type="external"><Parameter format="int" id="191221" name="id" value="121" /><Parameter format="str" id="191222" name="wintitle" value="Scope" /></Operation></ProcUnit></Project> No newline at end of file
1 <Project description="Testing USRP data reader" id="191" name="test01"><ReadUnit datatype="DigitalRF" id="1911" inputId="0" name="DigitalRFReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="DigitalRF" /><Parameter format="str" id="191112" name="path" value="/media/jchavez/DATA/mocked_data" /><Parameter format="date" id="191113" name="startDate" value="2000/07/03" /><Parameter format="date" id="191114" name="endDate" value="2017/07/03" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="Voltage" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /></ProcUnit></Project> No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,117 +1,117
1 1 #!/usr/bin/env python
2 2 '''
3 3 Created on Jul 7, 2014
4 4
5 5 @author: roj-idl71
6 6 '''
7 7 import os, sys
8 8
9 9 from schainpy.controller import Project
10 10
11 11 def main():
12 12
13 13 desc = "Testing USRP data reader"
14 14 filename = "schain.xml"
15 15 figpath = "./"
16 16 remotefolder = "/home/wmaster/graficos"
17 17
18 18 #this controller object save all user configuration and then execute each module
19 19 #with their parameters.
20 20 controllerObj = Project()
21 21
22 22 controllerObj.setup(id = '191', name='test01', description=desc)
23 23
24 24 #Creating a reader object with its parameters
25 25 #schainpy.model.io.jroIO_usrp.USRPReader.setup()
26 26 readUnitConfObj = controllerObj.addReadUnit(datatype='DigitalRF',
27 path='/home/jchavez/jicamarca/mocked_data/',
27 path='/media/jchavez/DATA/mocked_data',
28 28 startDate='2000/07/03',
29 29 endDate='2017/07/03',
30 30 startTime='00:00:00',
31 31 endTime='23:59:59',
32 32 online=0)
33 33
34 34 procUnitConfObj0 = controllerObj.addProcUnit(datatype='Voltage',
35 35 inputId=readUnitConfObj.getId())
36 36
37 37 # opObj10 = procUnitConfObj0.addOperation(name='selectHeights')
38 38 # opObj10.addParameter(name='minHei', value='0', format='float')
39 39 # opObj10.addParameter(name='maxHei', value='8', format='float')
40 40
41 41 # opObj10 = procUnitConfObj0.addOperation(name='setH0')
42 42 # opObj10.addParameter(name='h0', value='5.4', format='float')
43 43
44 44 # opObj10 = procUnitConfObj0.addOperation(name='Decoder', optype='external')
45 45 # opObj10.addParameter(name='code', value='1,-1', format='intlist')
46 46 # opObj10.addParameter(name='nCode', value='2', format='float')
47 47 # opObj10.addParameter(name='nBaud', value='1', format='float')
48 48
49 49 # opObj10 = procUnitConfObj0.addOperation(name='CohInt', optype='external')
50 # opObj10.addParameter(name='n', value='1', format='float')
50 # opObj10.addParameter(name='n', value='128', format='float')
51 51
52 opObj11 = procUnitConfObj0.addOperation(name='Scope', optype='external')
53 opObj11.addParameter(name='id', value='121', format='int')
54 opObj11.addParameter(name='wintitle', value='Scope', format='str')
52 # opObj11 = procUnitConfObj0.addOperation(name='Scope', optype='external')
53 # opObj11.addParameter(name='id', value='121', format='int')
54 # opObj11.addParameter(name='wintitle', value='Scope', format='str')
55 55
56 56 # procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra',
57 57 # inputId=procUnitConfObj0.getId())
58 58
59 59 # #Creating a processing object with its parameters
60 60 # #schainpy.model.proc.jroproc_spectra.SpectraProc.run()
61 61 # #If you need to add more parameters can use the "addParameter method"
62 62 # procUnitConfObj1.addParameter(name='nFFTPoints', value='8', format='int')
63 63 # procUnitConfObj1.addParameter(name='pairsList', value='(0,1)', format='pairslist')
64 64
65 65 # opObj10 = procUnitConfObj1.addOperation(name='IncohInt', optype='external')
66 66 # opObj10.addParameter(name='n', value='2', format='float')
67 67 #
68 68 #Using internal methods
69 69 #schainpy.model.proc.jroproc_spectra.SpectraProc.selectChannels()
70 70 # opObj10 = procUnitConfObj1.addOperation(name='selectChannels')
71 71 # opObj10.addParameter(name='channelList', value='0,1', format='intlist')
72 72
73 73 #Using internal methods
74 74 #schainpy.model.proc.jroproc_spectra.SpectraProc.selectHeights()
75 75 # opObj10 = procUnitConfObj1.addOperation(name='selectHeights')
76 76 # opObj10.addParameter(name='minHei', value='90', format='float')
77 77 # opObj10.addParameter(name='maxHei', value='180', format='float')
78 78
79 79 #Using external methods (new modules)
80 80 # #schainpy.model.proc.jroproc_spectra.IncohInt.setup()
81 81 # opObj12 = procUnitConfObj1.addOperation(name='IncohInt', optype='other')
82 82 # opObj12.addParameter(name='n', value='1', format='int')
83 83
84 84 #Using external methods (new modules)
85 85 #schainpy.model.graphics.jroplot_spectra.SpectraPlot.setup()
86 86 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
87 87 # opObj11.addParameter(name='id', value='11', format='int')
88 88 # opObj11.addParameter(name='wintitle', value='SpectraPlot', format='str')
89 89 # opObj11.addParameter(name='zmin', value='0', format='int')
90 90 # opObj11.addParameter(name='zmax', value='90', format='int')
91 91 # opObj11.addParameter(name='save', value='1', format='int')
92 92 # opObj11.addParameter(name='xmin', value='-20', format='float')
93 93 # opObj11.addParameter(name='xmax', value='20', format='float')
94 94
95 95 #Using external methods (new modules)
96 96 #schainpy.model.graphics.jroplot_spectra.RTIPlot.setup()
97 97 # opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
98 98 # opObj11.addParameter(name='id', value='30', format='int')
99 99 # opObj11.addParameter(name='wintitle', value='RTI', format='str')
100 100 # # opObj11.addParameter(name='zmin', value='0', format='int')
101 101 # # opObj11.addParameter(name='zmax', value='90', format='int')
102 102 # opObj11.addParameter(name='showprofile', value='1', format='int')
103 103 # opObj11.addParameter(name='timerange', value=str(2*60*60), format='int')
104 104 # opObj11.addParameter(name='xmin', value='19.5', format='float')
105 105 # opObj11.addParameter(name='xmax', value='20', format='float')
106 106
107 107 # opObj11 = procUnitConfObj1.addOperation(name='CrossSpectraPlot', optype='other')
108 108 # opObj11.addParameter(name='id', value='3', format='int')
109 109 # opObj11.addParameter(name='wintitle', value='CrossSpectraPlot', format='str')
110 110 # opObj11.addParameter(name='zmin', value='30', format='int')
111 111 # opObj11.addParameter(name='zmax', value='120', format='int')
112 112 # opObj11.addParameter(name='pairsList', value='(0,1)', format='pairslist')
113 113
114 114 controllerObj.start()
115 115
116 116 if __name__ == '__main__':
117 117 main()
General Comments 0
You need to be logged in to leave comments. Login now