##// END OF EJS Templates
schain
RSS
voltage reader enhancements
Julio Valdez -
r834:ee9a56ecbcc7
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,621 +1,632
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6
7 7 import numpy
8 8
9 9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
11 11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 12 from schainpy.model.data.jrodata import Voltage
13 # from _sha import blocksize
13 14
14 15 class VoltageReader(JRODataReader, ProcessingUnit):
15 16 """
16 17 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
17 18 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
18 19 perfiles*alturas*canales) son almacenados en la variable "buffer".
19 20
20 21 perfiles * alturas * canales
21 22
22 23 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
23 24 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
24 25 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
25 26 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
26 27
27 28 Example:
28 29
29 30 dpath = "/home/myuser/data"
30 31
31 32 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
32 33
33 34 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
34 35
35 36 readerObj = VoltageReader()
36 37
37 38 readerObj.setup(dpath, startTime, endTime)
38 39
39 40 while(True):
40 41
41 42 #to get one profile
42 43 profile = readerObj.getData()
43 44
44 45 #print the profile
45 46 print profile
46 47
47 48 #If you want to see all datablock
48 49 print readerObj.datablock
49 50
50 51 if readerObj.flagNoMoreFiles:
51 52 break
52 53
53 54 """
54 55
55 56 ext = ".r"
56 57
57 58 optchar = "D"
58 59 dataOut = None
59 60
60 61 def __init__(self):
61 62 """
62 63 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
63 64
64 65 Input:
65 66 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
66 67 almacenar un perfil de datos cada vez que se haga un requerimiento
67 68 (getData). El perfil sera obtenido a partir del buffer de datos,
68 69 si el buffer esta vacio se hara un nuevo proceso de lectura de un
69 70 bloque de datos.
70 71 Si este parametro no es pasado se creara uno internamente.
71 72
72 73 Variables afectadas:
73 74 self.dataOut
74 75
75 76 Return:
76 77 None
77 78 """
78 79
79 80 ProcessingUnit.__init__(self)
80 81
81 82 self.isConfig = False
82 83
83 84 self.datablock = None
84 85
85 86 self.utc = 0
86 87
87 88 self.ext = ".r"
88 89
89 90 self.optchar = "D"
90 91
91 92 self.basicHeaderObj = BasicHeader(LOCALTIME)
92 93
93 94 self.systemHeaderObj = SystemHeader()
94 95
95 96 self.radarControllerHeaderObj = RadarControllerHeader()
96 97
97 98 self.processingHeaderObj = ProcessingHeader()
98 99
99 100 self.online = 0
100 101
101 102 self.fp = None
102 103
103 104 self.idFile = None
104 105
105 106 self.dtype = None
106 107
107 108 self.fileSizeByHeader = None
108 109
109 110 self.filenameList = []
110 111
111 112 self.filename = None
112 113
113 114 self.fileSize = None
114 115
115 116 self.firstHeaderSize = 0
116 117
117 118 self.basicHeaderSize = 24
118 119
119 120 self.pathList = []
120 121
121 122 self.filenameList = []
122 123
123 124 self.lastUTTime = 0
124 125
125 126 self.maxTimeStep = 30
126 127
127 128 self.flagNoMoreFiles = 0
128 129
129 130 self.set = 0
130 131
131 132 self.path = None
132 133
133 134 self.profileIndex = 2**32-1
134 135
135 136 self.delay = 3 #seconds
136 137
137 138 self.nTries = 3 #quantity tries
138 139
139 140 self.nFiles = 3 #number of files for searching
140 141
141 142 self.nReadBlocks = 0
142 143
143 144 self.flagIsNewFile = 1
144 145
145 146 self.__isFirstTimeOnline = 1
146 147
147 148 # self.ippSeconds = 0
148 149
149 150 self.flagDiscontinuousBlock = 0
150 151
151 152 self.flagIsNewBlock = 0
152 153
153 154 self.nTotalBlocks = 0
154 155
155 156 self.blocksize = 0
156 157
157 158 self.dataOut = self.createObjByDefault()
158 159
159 160 self.nTxs = 1
160 161
161 162 self.txIndex = 0
162 163
163 164 def createObjByDefault(self):
164 165
165 166 dataObj = Voltage()
166 167
167 168 return dataObj
168 169
169 170 def __hasNotDataInBuffer(self):
170 171
171 172 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
172 173 return 1
173 174
174 175 return 0
175 176
176 177
177 178 def getBlockDimension(self):
178 179 """
179 180 Obtiene la cantidad de puntos a leer por cada bloque de datos
180 181
181 182 Affected:
182 183 self.blocksize
183 184
184 185 Return:
185 186 None
186 187 """
187 188 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
188 189 self.blocksize = pts2read
189 190
190 191
191 192 def readBlock(self):
192 193 """
193 194 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
194 195 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
195 196 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
196 197 es seteado a 0
197 198
198 199 Inputs:
199 200 None
200 201
201 202 Return:
202 203 None
203 204
204 205 Affected:
205 206 self.profileIndex
206 207 self.datablock
207 208 self.flagIsNewFile
208 209 self.flagIsNewBlock
209 210 self.nTotalBlocks
210 211
211 212 Exceptions:
212 213 Si un bloque leido no es un bloque valido
213 214 """
214 215 current_pointer_location = self.fp.tell()
215 216 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
216 217
217 218 try:
218 219 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
219 220 except:
220 221 #print "The read block (%3d) has not enough data" %self.nReadBlocks
221 222
222 223 if self.waitDataBlock(pointer_location=current_pointer_location):
223 224 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
224 225 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
225 226 # return 0
226 227
227 228 #Dimensions : nChannels, nProfiles, nSamples
228 229
229 230 junk = numpy.transpose(junk, (2,0,1))
230 231 self.datablock = junk['real'] + junk['imag']*1j
231 232
232 233 self.profileIndex = 0
233 234
234 235 self.flagIsNewFile = 0
235 236 self.flagIsNewBlock = 1
236 237
237 238 self.nTotalBlocks += 1
238 239 self.nReadBlocks += 1
239 240
240 241 return 1
241 242
242 243 def getFirstHeader(self):
243 244
244 245 self.getBasicHeader()
245 246
246 247 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
247 248
248 249 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
249 250
250 251 if self.nTxs > 1:
251 252 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
252 253
253 254 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
254 255
255 256 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
256 257 #
257 258 # if self.radarControllerHeaderObj.code is not None:
258 259 #
259 260 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
260 261 #
261 262 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
262 263 #
263 264 # self.dataOut.code = self.radarControllerHeaderObj.code
264 265
265 266 self.dataOut.dtype = self.dtype
266 267
267 268 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
268 269
269 270 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
270 271
271 272 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
272 273
273 274 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
274 275
275 276 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
276 277
277 278 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
278 279
279 280 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
280 281
281 282 def reshapeData(self):
282 283
283 284 if self.nTxs < 0:
284 285 return
285 286
286 287 if self.nTxs == 1:
287 288 return
288 289
289 290 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
290 291 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
291 292
292 293 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
293 294 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
294 295
295 296 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
296 297
297 298 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
298 299 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
299 300 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
300 301
301 302 return
302 303
303 304 def getData(self):
304 305 """
305 306 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
306 307 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
307 308 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
308 309 "readNextBlock"
309 310
310 311 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
311 312
312 313 Return:
313 314
314 315 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
315 316 es igual al total de perfiles leidos desde el archivo.
316 317
317 318 Si self.getByBlock == False:
318 319
319 320 self.dataOut.data = buffer[:, thisProfile, :]
320 321
321 322 shape = [nChannels, nHeis]
322 323
323 324 Si self.getByBlock == True:
324 325
325 326 self.dataOut.data = buffer[:, :, :]
326 327
327 328 shape = [nChannels, nProfiles, nHeis]
328 329
329 330 Variables afectadas:
330 331 self.dataOut
331 332 self.profileIndex
332 333
333 334 Affected:
334 335 self.dataOut
335 336 self.profileIndex
336 337 self.flagDiscontinuousBlock
337 338 self.flagIsNewBlock
338 339 """
339 340
340 341 if self.flagNoMoreFiles:
341 342 self.dataOut.flagNoData = True
342 343 print 'Process finished'
343 344 return 0
344 345
345 346 self.flagDiscontinuousBlock = 0
346 347 self.flagIsNewBlock = 0
347 348
348 349 if self.__hasNotDataInBuffer():
349 350
350 351 if not( self.readNextBlock() ):
351 352 return 0
352 353
353 354 self.getFirstHeader()
354 355
355 356 self.reshapeData()
356 357
357 358 if self.datablock is None:
358 359 self.dataOut.flagNoData = True
359 360 return 0
360 361
361 362 if not self.getByBlock:
362 363
363 364 """
364 365 Return profile by profile
365 366
366 367 If nTxs > 1 then one profile is divided by nTxs and number of total
367 368 blocks is increased by nTxs (nProfiles *= nTxs)
368 369 """
369 370 self.dataOut.flagDataAsBlock = False
370 371 self.dataOut.data = self.datablock[:,self.profileIndex,:]
371 372 self.dataOut.profileIndex = self.profileIndex
372 373
373 374 self.profileIndex += 1
374 375
375 376 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
376 377 # """
377 378 # Return all block
378 379 # """
379 380 # self.dataOut.flagDataAsBlock = True
380 381 # self.dataOut.data = self.datablock
381 382 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
382 383 #
383 384 # self.profileIndex = self.dataOut.nProfiles
384 385
385 386 else:
386 387 """
387 388 Return a block
388 389 """
389 390 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
390 391 if self.selBlocktime != None: self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(self.dataOut.ippSeconds*self.dataOut.nProfiles)))
391 392
392 393 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
393 394 self.profileIndex += self.selBlocksize
395 datasize = self.dataOut.data.shape[1]
394 396
395 while self.dataOut.data.shape[1] < self.selBlocksize: #Not enough profiles to fill the block
397 if datasize < self.selBlocksize:
398 buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex')
399 buffer[:,:datasize,:] = self.dataOut.data
400
401 while datasize < self.selBlocksize: #Not enough profiles to fill the block
396 402 if not( self.readNextBlock() ):
397 403 return 0
398 404 self.getFirstHeader()
399 405 self.reshapeData()
400 406 if self.datablock is None:
401 407 self.dataOut.flagNoData = True
402 408 return 0
403 409 #stack data
404 indMax = self.selBlocksize - self.dataOut.data.shape[1]
405 self.dataOut.data = numpy.hstack((self.dataOut.data,self.datablock[:,:indMax,:]))
406 self.profileIndex = indMax
410 blockIndex = self.selBlocksize - datasize
411 datablock1 = self.datablock[:,:blockIndex,:]
412
413 buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1
414 datasize += datablock1.shape[1]
415
416 self.dataOut.data = buffer
417 self.profileIndex = blockIndex
407 418
408 419 self.dataOut.flagDataAsBlock = True
409 420 self.dataOut.nProfiles = self.selBlocksize
410 421
411 422 self.dataOut.flagNoData = False
412 423
413 424 self.getBasicHeader()
414 425
415 426 self.dataOut.realtime = self.online
416 427
417 428 return self.dataOut.data
418 429
419 430 class VoltageWriter(JRODataWriter, Operation):
420 431 """
421 432 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
422 433 de los datos siempre se realiza por bloques.
423 434 """
424 435
425 436 ext = ".r"
426 437
427 438 optchar = "D"
428 439
429 440 shapeBuffer = None
430 441
431 442
432 443 def __init__(self):
433 444 """
434 445 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
435 446
436 447 Affected:
437 448 self.dataOut
438 449
439 450 Return: None
440 451 """
441 452 Operation.__init__(self)
442 453
443 454 self.nTotalBlocks = 0
444 455
445 456 self.profileIndex = 0
446 457
447 458 self.isConfig = False
448 459
449 460 self.fp = None
450 461
451 462 self.flagIsNewFile = 1
452 463
453 464 self.blockIndex = 0
454 465
455 466 self.flagIsNewBlock = 0
456 467
457 468 self.setFile = None
458 469
459 470 self.dtype = None
460 471
461 472 self.path = None
462 473
463 474 self.filename = None
464 475
465 476 self.basicHeaderObj = BasicHeader(LOCALTIME)
466 477
467 478 self.systemHeaderObj = SystemHeader()
468 479
469 480 self.radarControllerHeaderObj = RadarControllerHeader()
470 481
471 482 self.processingHeaderObj = ProcessingHeader()
472 483
473 484 def hasAllDataInBuffer(self):
474 485 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
475 486 return 1
476 487 return 0
477 488
478 489
479 490 def setBlockDimension(self):
480 491 """
481 492 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
482 493
483 494 Affected:
484 495 self.shape_spc_Buffer
485 496 self.shape_cspc_Buffer
486 497 self.shape_dc_Buffer
487 498
488 499 Return: None
489 500 """
490 501 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
491 502 self.processingHeaderObj.nHeights,
492 503 self.systemHeaderObj.nChannels)
493 504
494 505 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
495 506 self.processingHeaderObj.profilesPerBlock,
496 507 self.processingHeaderObj.nHeights),
497 508 dtype=numpy.dtype('complex64'))
498 509
499 510 def writeBlock(self):
500 511 """
501 512 Escribe el buffer en el file designado
502 513
503 514 Affected:
504 515 self.profileIndex
505 516 self.flagIsNewFile
506 517 self.flagIsNewBlock
507 518 self.nTotalBlocks
508 519 self.blockIndex
509 520
510 521 Return: None
511 522 """
512 523 data = numpy.zeros( self.shapeBuffer, self.dtype )
513 524
514 525 junk = numpy.transpose(self.datablock, (1,2,0))
515 526
516 527 data['real'] = junk.real
517 528 data['imag'] = junk.imag
518 529
519 530 data = data.reshape( (-1) )
520 531
521 532 data.tofile( self.fp )
522 533
523 534 self.datablock.fill(0)
524 535
525 536 self.profileIndex = 0
526 537 self.flagIsNewFile = 0
527 538 self.flagIsNewBlock = 1
528 539
529 540 self.blockIndex += 1
530 541 self.nTotalBlocks += 1
531 542
532 543 # print "[Writing] Block = %04d" %self.blockIndex
533 544
534 545 def putData(self):
535 546 """
536 547 Setea un bloque de datos y luego los escribe en un file
537 548
538 549 Affected:
539 550 self.flagIsNewBlock
540 551 self.profileIndex
541 552
542 553 Return:
543 554 0 : Si no hay data o no hay mas files que puedan escribirse
544 555 1 : Si se escribio la data de un bloque en un file
545 556 """
546 557 if self.dataOut.flagNoData:
547 558 return 0
548 559
549 560 self.flagIsNewBlock = 0
550 561
551 562 if self.dataOut.flagDiscontinuousBlock:
552 563 self.datablock.fill(0)
553 564 self.profileIndex = 0
554 565 self.setNextFile()
555 566
556 567 if self.profileIndex == 0:
557 568 self.setBasicHeader()
558 569
559 570 self.datablock[:,self.profileIndex,:] = self.dataOut.data
560 571
561 572 self.profileIndex += 1
562 573
563 574 if self.hasAllDataInBuffer():
564 575 #if self.flagIsNewFile:
565 576 self.writeNextBlock()
566 577 # self.setFirstHeader()
567 578
568 579 return 1
569 580
570 581 def __getBlockSize(self):
571 582 '''
572 583 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
573 584 '''
574 585
575 586 dtype_width = self.getDtypeWidth()
576 587
577 588 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
578 589
579 590 return blocksize
580 591
581 592 def setFirstHeader(self):
582 593
583 594 """
584 595 Obtiene una copia del First Header
585 596
586 597 Affected:
587 598 self.systemHeaderObj
588 599 self.radarControllerHeaderObj
589 600 self.dtype
590 601
591 602 Return:
592 603 None
593 604 """
594 605
595 606 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
596 607 self.systemHeaderObj.nChannels = self.dataOut.nChannels
597 608 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
598 609
599 610 self.processingHeaderObj.dtype = 0 # Voltage
600 611 self.processingHeaderObj.blockSize = self.__getBlockSize()
601 612 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
602 613 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
603 614 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
604 615 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
605 616 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
606 617 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
607 618
608 619 if self.dataOut.code is not None:
609 620 self.processingHeaderObj.code = self.dataOut.code
610 621 self.processingHeaderObj.nCode = self.dataOut.nCode
611 622 self.processingHeaderObj.nBaud = self.dataOut.nBaud
612 623
613 624 if self.processingHeaderObj.nWindows != 0:
614 625 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
615 626 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
616 627 self.processingHeaderObj.nHeights = self.dataOut.nHeights
617 628 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
618 629
619 630 self.processingHeaderObj.processFlags = self.getProcessFlags()
620 631
621 632 self.setBasicHeader() No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now