##// END OF EJS Templates
schain
RSS
Correction to jroIO_voltage when reading by block
Julio Valdez -
r849:44c4b8db71a1
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,632 +1,637
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6
7 7 import numpy
8 8
9 9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
11 11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 12 from schainpy.model.data.jrodata import Voltage
13 13 # from _sha import blocksize
14 14
15 15 class VoltageReader(JRODataReader, ProcessingUnit):
16 16 """
17 17 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
18 18 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
19 19 perfiles*alturas*canales) son almacenados en la variable "buffer".
20 20
21 21 perfiles * alturas * canales
22 22
23 23 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
24 24 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
25 25 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
26 26 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
27 27
28 28 Example:
29 29
30 30 dpath = "/home/myuser/data"
31 31
32 32 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
33 33
34 34 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
35 35
36 36 readerObj = VoltageReader()
37 37
38 38 readerObj.setup(dpath, startTime, endTime)
39 39
40 40 while(True):
41 41
42 42 #to get one profile
43 43 profile = readerObj.getData()
44 44
45 45 #print the profile
46 46 print profile
47 47
48 48 #If you want to see all datablock
49 49 print readerObj.datablock
50 50
51 51 if readerObj.flagNoMoreFiles:
52 52 break
53 53
54 54 """
55 55
56 56 ext = ".r"
57 57
58 58 optchar = "D"
59 59 dataOut = None
60 60
61 61 def __init__(self):
62 62 """
63 63 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
64 64
65 65 Input:
66 66 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
67 67 almacenar un perfil de datos cada vez que se haga un requerimiento
68 68 (getData). El perfil sera obtenido a partir del buffer de datos,
69 69 si el buffer esta vacio se hara un nuevo proceso de lectura de un
70 70 bloque de datos.
71 71 Si este parametro no es pasado se creara uno internamente.
72 72
73 73 Variables afectadas:
74 74 self.dataOut
75 75
76 76 Return:
77 77 None
78 78 """
79 79
80 80 ProcessingUnit.__init__(self)
81 81
82 82 self.isConfig = False
83 83
84 84 self.datablock = None
85 85
86 86 self.utc = 0
87 87
88 88 self.ext = ".r"
89 89
90 90 self.optchar = "D"
91 91
92 92 self.basicHeaderObj = BasicHeader(LOCALTIME)
93 93
94 94 self.systemHeaderObj = SystemHeader()
95 95
96 96 self.radarControllerHeaderObj = RadarControllerHeader()
97 97
98 98 self.processingHeaderObj = ProcessingHeader()
99 99
100 100 self.online = 0
101 101
102 102 self.fp = None
103 103
104 104 self.idFile = None
105 105
106 106 self.dtype = None
107 107
108 108 self.fileSizeByHeader = None
109 109
110 110 self.filenameList = []
111 111
112 112 self.filename = None
113 113
114 114 self.fileSize = None
115 115
116 116 self.firstHeaderSize = 0
117 117
118 118 self.basicHeaderSize = 24
119 119
120 120 self.pathList = []
121 121
122 122 self.filenameList = []
123 123
124 124 self.lastUTTime = 0
125 125
126 126 self.maxTimeStep = 30
127 127
128 128 self.flagNoMoreFiles = 0
129 129
130 130 self.set = 0
131 131
132 132 self.path = None
133 133
134 134 self.profileIndex = 2**32-1
135 135
136 136 self.delay = 3 #seconds
137 137
138 138 self.nTries = 3 #quantity tries
139 139
140 140 self.nFiles = 3 #number of files for searching
141 141
142 142 self.nReadBlocks = 0
143 143
144 144 self.flagIsNewFile = 1
145 145
146 146 self.__isFirstTimeOnline = 1
147 147
148 148 # self.ippSeconds = 0
149 149
150 150 self.flagDiscontinuousBlock = 0
151 151
152 152 self.flagIsNewBlock = 0
153 153
154 154 self.nTotalBlocks = 0
155 155
156 156 self.blocksize = 0
157 157
158 158 self.dataOut = self.createObjByDefault()
159 159
160 160 self.nTxs = 1
161 161
162 162 self.txIndex = 0
163 163
164 164 def createObjByDefault(self):
165 165
166 166 dataObj = Voltage()
167 167
168 168 return dataObj
169 169
170 170 def __hasNotDataInBuffer(self):
171 171
172 172 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
173 173 return 1
174 174
175 175 return 0
176 176
177 177
178 178 def getBlockDimension(self):
179 179 """
180 180 Obtiene la cantidad de puntos a leer por cada bloque de datos
181 181
182 182 Affected:
183 183 self.blocksize
184 184
185 185 Return:
186 186 None
187 187 """
188 188 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
189 189 self.blocksize = pts2read
190 190
191 191
192 192 def readBlock(self):
193 193 """
194 194 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
195 195 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
196 196 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
197 197 es seteado a 0
198 198
199 199 Inputs:
200 200 None
201 201
202 202 Return:
203 203 None
204 204
205 205 Affected:
206 206 self.profileIndex
207 207 self.datablock
208 208 self.flagIsNewFile
209 209 self.flagIsNewBlock
210 210 self.nTotalBlocks
211 211
212 212 Exceptions:
213 213 Si un bloque leido no es un bloque valido
214 214 """
215 215 current_pointer_location = self.fp.tell()
216 216 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
217 217
218 218 try:
219 219 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
220 220 except:
221 221 #print "The read block (%3d) has not enough data" %self.nReadBlocks
222 222
223 223 if self.waitDataBlock(pointer_location=current_pointer_location):
224 224 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
225 225 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
226 226 # return 0
227 227
228 228 #Dimensions : nChannels, nProfiles, nSamples
229 229
230 230 junk = numpy.transpose(junk, (2,0,1))
231 231 self.datablock = junk['real'] + junk['imag']*1j
232 232
233 233 self.profileIndex = 0
234 234
235 235 self.flagIsNewFile = 0
236 236 self.flagIsNewBlock = 1
237 237
238 238 self.nTotalBlocks += 1
239 239 self.nReadBlocks += 1
240 240
241 241 return 1
242 242
243 243 def getFirstHeader(self):
244 244
245 245 self.getBasicHeader()
246 246
247 247 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
248 248
249 249 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
250 250
251 251 if self.nTxs > 1:
252 252 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
253 253
254 254 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
255 255
256 256 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
257 257 #
258 258 # if self.radarControllerHeaderObj.code is not None:
259 259 #
260 260 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
261 261 #
262 262 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
263 263 #
264 264 # self.dataOut.code = self.radarControllerHeaderObj.code
265 265
266 266 self.dataOut.dtype = self.dtype
267 267
268 268 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
269 269
270 270 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
271 271
272 272 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
273 273
274 274 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
275 275
276 276 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
277 277
278 278 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
279 279
280 280 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
281 281
282 282 def reshapeData(self):
283 283
284 284 if self.nTxs < 0:
285 285 return
286 286
287 287 if self.nTxs == 1:
288 288 return
289 289
290 290 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
291 291 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
292 292
293 293 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
294 294 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
295 295
296 296 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
297 297
298 298 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
299 299 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
300 300 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
301 301
302 302 return
303 303
304 304 def getData(self):
305 305 """
306 306 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
307 307 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
308 308 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
309 309 "readNextBlock"
310 310
311 311 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
312 312
313 313 Return:
314 314
315 315 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
316 316 es igual al total de perfiles leidos desde el archivo.
317 317
318 318 Si self.getByBlock == False:
319 319
320 320 self.dataOut.data = buffer[:, thisProfile, :]
321 321
322 322 shape = [nChannels, nHeis]
323 323
324 324 Si self.getByBlock == True:
325 325
326 326 self.dataOut.data = buffer[:, :, :]
327 327
328 328 shape = [nChannels, nProfiles, nHeis]
329 329
330 330 Variables afectadas:
331 331 self.dataOut
332 332 self.profileIndex
333 333
334 334 Affected:
335 335 self.dataOut
336 336 self.profileIndex
337 337 self.flagDiscontinuousBlock
338 338 self.flagIsNewBlock
339 339 """
340 340
341 341 if self.flagNoMoreFiles:
342 342 self.dataOut.flagNoData = True
343 343 print 'Process finished'
344 344 return 0
345 345
346 346 self.flagDiscontinuousBlock = 0
347 347 self.flagIsNewBlock = 0
348 348
349 349 if self.__hasNotDataInBuffer():
350 350
351 351 if not( self.readNextBlock() ):
352 352 return 0
353 353
354 354 self.getFirstHeader()
355 355
356 356 self.reshapeData()
357 357
358 358 if self.datablock is None:
359 359 self.dataOut.flagNoData = True
360 360 return 0
361 361
362 362 if not self.getByBlock:
363 363
364 364 """
365 365 Return profile by profile
366 366
367 367 If nTxs > 1 then one profile is divided by nTxs and number of total
368 368 blocks is increased by nTxs (nProfiles *= nTxs)
369 369 """
370 370 self.dataOut.flagDataAsBlock = False
371 371 self.dataOut.data = self.datablock[:,self.profileIndex,:]
372 372 self.dataOut.profileIndex = self.profileIndex
373 373
374 374 self.profileIndex += 1
375 375
376 376 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
377 377 # """
378 378 # Return all block
379 379 # """
380 380 # self.dataOut.flagDataAsBlock = True
381 381 # self.dataOut.data = self.datablock
382 382 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
383 383 #
384 384 # self.profileIndex = self.dataOut.nProfiles
385 385
386 386 else:
387 387 """
388 388 Return a block
389 389 """
390 390 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
391 if self.selBlocktime != None: self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(self.dataOut.ippSeconds*self.dataOut.nProfiles)))
391 if self.selBlocktime != None:
392 if self.dataOut.nCohInt is not None:
393 nCohInt = self.dataOut.nCohInt
394 else:
395 nCohInt = 1
396 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
392 397
393 398 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
394 399 self.profileIndex += self.selBlocksize
395 400 datasize = self.dataOut.data.shape[1]
396 401
397 402 if datasize < self.selBlocksize:
398 403 buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex')
399 404 buffer[:,:datasize,:] = self.dataOut.data
400 405
401 406 while datasize < self.selBlocksize: #Not enough profiles to fill the block
402 407 if not( self.readNextBlock() ):
403 408 return 0
404 409 self.getFirstHeader()
405 410 self.reshapeData()
406 411 if self.datablock is None:
407 412 self.dataOut.flagNoData = True
408 413 return 0
409 414 #stack data
410 415 blockIndex = self.selBlocksize - datasize
411 416 datablock1 = self.datablock[:,:blockIndex,:]
412 417
413 418 buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1
414 419 datasize += datablock1.shape[1]
415 420
416 421 self.dataOut.data = buffer
417 422 self.profileIndex = blockIndex
418 423
419 424 self.dataOut.flagDataAsBlock = True
420 425 self.dataOut.nProfiles = self.dataOut.data.shape[1]
421 426
422 427 self.dataOut.flagNoData = False
423 428
424 429 self.getBasicHeader()
425 430
426 431 self.dataOut.realtime = self.online
427 432
428 433 return self.dataOut.data
429 434
430 435 class VoltageWriter(JRODataWriter, Operation):
431 436 """
432 437 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
433 438 de los datos siempre se realiza por bloques.
434 439 """
435 440
436 441 ext = ".r"
437 442
438 443 optchar = "D"
439 444
440 445 shapeBuffer = None
441 446
442 447
443 448 def __init__(self):
444 449 """
445 450 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
446 451
447 452 Affected:
448 453 self.dataOut
449 454
450 455 Return: None
451 456 """
452 457 Operation.__init__(self)
453 458
454 459 self.nTotalBlocks = 0
455 460
456 461 self.profileIndex = 0
457 462
458 463 self.isConfig = False
459 464
460 465 self.fp = None
461 466
462 467 self.flagIsNewFile = 1
463 468
464 469 self.blockIndex = 0
465 470
466 471 self.flagIsNewBlock = 0
467 472
468 473 self.setFile = None
469 474
470 475 self.dtype = None
471 476
472 477 self.path = None
473 478
474 479 self.filename = None
475 480
476 481 self.basicHeaderObj = BasicHeader(LOCALTIME)
477 482
478 483 self.systemHeaderObj = SystemHeader()
479 484
480 485 self.radarControllerHeaderObj = RadarControllerHeader()
481 486
482 487 self.processingHeaderObj = ProcessingHeader()
483 488
484 489 def hasAllDataInBuffer(self):
485 490 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
486 491 return 1
487 492 return 0
488 493
489 494
490 495 def setBlockDimension(self):
491 496 """
492 497 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
493 498
494 499 Affected:
495 500 self.shape_spc_Buffer
496 501 self.shape_cspc_Buffer
497 502 self.shape_dc_Buffer
498 503
499 504 Return: None
500 505 """
501 506 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
502 507 self.processingHeaderObj.nHeights,
503 508 self.systemHeaderObj.nChannels)
504 509
505 510 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
506 511 self.processingHeaderObj.profilesPerBlock,
507 512 self.processingHeaderObj.nHeights),
508 513 dtype=numpy.dtype('complex64'))
509 514
510 515 def writeBlock(self):
511 516 """
512 517 Escribe el buffer en el file designado
513 518
514 519 Affected:
515 520 self.profileIndex
516 521 self.flagIsNewFile
517 522 self.flagIsNewBlock
518 523 self.nTotalBlocks
519 524 self.blockIndex
520 525
521 526 Return: None
522 527 """
523 528 data = numpy.zeros( self.shapeBuffer, self.dtype )
524 529
525 530 junk = numpy.transpose(self.datablock, (1,2,0))
526 531
527 532 data['real'] = junk.real
528 533 data['imag'] = junk.imag
529 534
530 535 data = data.reshape( (-1) )
531 536
532 537 data.tofile( self.fp )
533 538
534 539 self.datablock.fill(0)
535 540
536 541 self.profileIndex = 0
537 542 self.flagIsNewFile = 0
538 543 self.flagIsNewBlock = 1
539 544
540 545 self.blockIndex += 1
541 546 self.nTotalBlocks += 1
542 547
543 548 # print "[Writing] Block = %04d" %self.blockIndex
544 549
545 550 def putData(self):
546 551 """
547 552 Setea un bloque de datos y luego los escribe en un file
548 553
549 554 Affected:
550 555 self.flagIsNewBlock
551 556 self.profileIndex
552 557
553 558 Return:
554 559 0 : Si no hay data o no hay mas files que puedan escribirse
555 560 1 : Si se escribio la data de un bloque en un file
556 561 """
557 562 if self.dataOut.flagNoData:
558 563 return 0
559 564
560 565 self.flagIsNewBlock = 0
561 566
562 567 if self.dataOut.flagDiscontinuousBlock:
563 568 self.datablock.fill(0)
564 569 self.profileIndex = 0
565 570 self.setNextFile()
566 571
567 572 if self.profileIndex == 0:
568 573 self.setBasicHeader()
569 574
570 575 self.datablock[:,self.profileIndex,:] = self.dataOut.data
571 576
572 577 self.profileIndex += 1
573 578
574 579 if self.hasAllDataInBuffer():
575 580 #if self.flagIsNewFile:
576 581 self.writeNextBlock()
577 582 # self.setFirstHeader()
578 583
579 584 return 1
580 585
581 586 def __getBlockSize(self):
582 587 '''
583 588 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
584 589 '''
585 590
586 591 dtype_width = self.getDtypeWidth()
587 592
588 593 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
589 594
590 595 return blocksize
591 596
592 597 def setFirstHeader(self):
593 598
594 599 """
595 600 Obtiene una copia del First Header
596 601
597 602 Affected:
598 603 self.systemHeaderObj
599 604 self.radarControllerHeaderObj
600 605 self.dtype
601 606
602 607 Return:
603 608 None
604 609 """
605 610
606 611 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
607 612 self.systemHeaderObj.nChannels = self.dataOut.nChannels
608 613 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
609 614
610 615 self.processingHeaderObj.dtype = 0 # Voltage
611 616 self.processingHeaderObj.blockSize = self.__getBlockSize()
612 617 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
613 618 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
614 619 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
615 620 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
616 621 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
617 622 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
618 623
619 624 if self.dataOut.code is not None:
620 625 self.processingHeaderObj.code = self.dataOut.code
621 626 self.processingHeaderObj.nCode = self.dataOut.nCode
622 627 self.processingHeaderObj.nBaud = self.dataOut.nBaud
623 628
624 629 if self.processingHeaderObj.nWindows != 0:
625 630 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
626 631 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
627 632 self.processingHeaderObj.nHeights = self.dataOut.nHeights
628 633 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
629 634
630 635 self.processingHeaderObj.processFlags = self.getProcessFlags()
631 636
632 637 self.setBasicHeader() No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now