##// END OF EJS Templates
schain-jc
RSS
Alexander Valdez -
r562:d00bee51f851
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,650 +1,650
1 1 '''
2 2
3 3 '''
4 4 import numpy
5 5
6 6 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
7 7 from model.proc.jroproc_base import ProcessingUnit, Operation
8 8 from model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
9 9 from model.data.jrodata import Voltage
10 10
11 11 class VoltageReader(JRODataReader, ProcessingUnit):
12 12 """
13 13 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
14 14 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
15 15 perfiles*alturas*canales) son almacenados en la variable "buffer".
16 16
17 17 perfiles * alturas * canales
18 18
19 19 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
20 20 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
21 21 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
22 22 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
23 23
24 24 Example:
25 25
26 26 dpath = "/home/myuser/data"
27 27
28 28 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
29 29
30 30 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
31 31
32 32 readerObj = VoltageReader()
33 33
34 34 readerObj.setup(dpath, startTime, endTime)
35 35
36 36 while(True):
37 37
38 38 #to get one profile
39 39 profile = readerObj.getData()
40 40
41 41 #print the profile
42 42 print profile
43 43
44 44 #If you want to see all datablock
45 45 print readerObj.datablock
46 46
47 47 if readerObj.flagNoMoreFiles:
48 48 break
49 49
50 50 """
51 51
52 52 ext = ".r"
53 53
54 54 optchar = "D"
55 55 dataOut = None
56 56
57 57
58 58 def __init__(self):
59 59 """
60 60 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
61 61
62 62 Input:
63 63 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
64 64 almacenar un perfil de datos cada vez que se haga un requerimiento
65 65 (getData). El perfil sera obtenido a partir del buffer de datos,
66 66 si el buffer esta vacio se hara un nuevo proceso de lectura de un
67 67 bloque de datos.
68 68 Si este parametro no es pasado se creara uno internamente.
69 69
70 70 Variables afectadas:
71 71 self.dataOut
72 72
73 73 Return:
74 74 None
75 75 """
76 76
77 77 ProcessingUnit.__init__(self)
78 78
79 79 self.isConfig = False
80 80
81 81 self.datablock = None
82 82
83 83 self.utc = 0
84 84
85 85 self.ext = ".r"
86 86
87 87 self.optchar = "D"
88 88
89 89 self.basicHeaderObj = BasicHeader(LOCALTIME)
90 90
91 91 self.systemHeaderObj = SystemHeader()
92 92
93 93 self.radarControllerHeaderObj = RadarControllerHeader()
94 94
95 95 self.processingHeaderObj = ProcessingHeader()
96 96
97 97 self.online = 0
98 98
99 99 self.fp = None
100 100
101 101 self.idFile = None
102 102
103 103 self.dtype = None
104 104
105 105 self.fileSizeByHeader = None
106 106
107 107 self.filenameList = []
108 108
109 109 self.filename = None
110 110
111 111 self.fileSize = None
112 112
113 113 self.firstHeaderSize = 0
114 114
115 115 self.basicHeaderSize = 24
116 116
117 117 self.pathList = []
118 118
119 119 self.filenameList = []
120 120
121 121 self.lastUTTime = 0
122 122
123 123 self.maxTimeStep = 30
124 124
125 125 self.flagNoMoreFiles = 0
126 126
127 127 self.set = 0
128 128
129 129 self.path = None
130 130
131 131 self.profileIndex = 2**32-1
132 132
133 133 self.delay = 3 #seconds
134 134
135 135 self.nTries = 3 #quantity tries
136 136
137 137 self.nFiles = 3 #number of files for searching
138 138
139 139 self.nReadBlocks = 0
140 140
141 141 self.flagIsNewFile = 1
142 142
143 143 self.__isFirstTimeOnline = 1
144 144
145 145 # self.ippSeconds = 0
146 146
147 147 self.flagTimeBlock = 0
148 148
149 149 self.flagIsNewBlock = 0
150 150
151 151 self.nTotalBlocks = 0
152 152
153 153 self.blocksize = 0
154 154
155 155 self.dataOut = self.createObjByDefault()
156 156
157 157 self.nTxs = 1
158 158
159 159 self.txIndex = 0
160 160
161 161 def createObjByDefault(self):
162 162
163 163 dataObj = Voltage()
164 164
165 165 return dataObj
166 166
167 167 def __hasNotDataInBuffer(self):
168 168
169 169 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
170 170 return 1
171 171
172 172 return 0
173 173
174 174
175 175 def getBlockDimension(self):
176 176 """
177 177 Obtiene la cantidad de puntos a leer por cada bloque de datos
178 178
179 179 Affected:
180 180 self.blocksize
181 181
182 182 Return:
183 183 None
184 184 """
185 185 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
186 186 self.blocksize = pts2read
187 187
188 188
189 189 def readBlock(self):
190 190 """
191 191 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
192 192 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
193 193 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
194 194 es seteado a 0
195 195
196 196 Inputs:
197 197 None
198 198
199 199 Return:
200 200 None
201 201
202 202 Affected:
203 203 self.profileIndex
204 204 self.datablock
205 205 self.flagIsNewFile
206 206 self.flagIsNewBlock
207 207 self.nTotalBlocks
208 208
209 209 Exceptions:
210 210 Si un bloque leido no es un bloque valido
211 211 """
212 212 current_pointer_location = self.fp.tell()
213 213 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
214 214
215 215 try:
216 216 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
217 217 except:
218 218 #print "The read block (%3d) has not enough data" %self.nReadBlocks
219 219
220 220 if self.waitDataBlock(pointer_location=current_pointer_location):
221 221 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
222 222 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
223 223 # return 0
224 224
225 225 junk = numpy.transpose(junk, (2,0,1))
226 226 self.datablock = junk['real'] + junk['imag']*1j
227 227
228 228 self.profileIndex = 0
229 229
230 230 self.flagIsNewFile = 0
231 231 self.flagIsNewBlock = 1
232 232
233 233 self.nTotalBlocks += 1
234 234 self.nReadBlocks += 1
235 235
236 236 return 1
237 237
238 238 def getFirstHeader(self):
239 239
240 240 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
241 241
242 242 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
243 243
244 244 if self.nTxs > 1:
245 self.dataOut.radarControllerHeaderObj.ippSeconds /= self.nTxs
245 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
246 246
247 247 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
248 248
249 249 if self.radarControllerHeaderObj.code != None:
250 250
251 251 self.dataOut.nCode = self.radarControllerHeaderObj.nCode
252 252
253 253 self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
254 254
255 255 self.dataOut.code = self.radarControllerHeaderObj.code
256 256
257 257 self.dataOut.dtype = self.dtype
258 258
259 259 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
260 260
261 261 if self.processingHeaderObj.nHeights % self.nTxs != 0:
262 262 raise ValueError, "nTxs (%d) should be a multiple of nHeights (%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
263 263
264 264 xf = self.processingHeaderObj.firstHeight + int(self.processingHeaderObj.nHeights/self.nTxs)*self.processingHeaderObj.deltaHeight
265 265
266 266 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
267 267
268 268 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
269 269
270 270 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
271 271
272 272 self.dataOut.flagShiftFFT = False
273 273
274 274 self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
275 275
276 276 self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip
277 277
278 278 self.dataOut.flagShiftFFT = False
279 279
280 280 def getData(self):
281 281 """
282 282 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
283 283 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
284 284 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
285 285 "readNextBlock"
286 286
287 287 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
288 288
289 289 Return:
290 290
291 291 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
292 292 es igual al total de perfiles leidos desde el archivo.
293 293
294 294 Si self.getByBlock == False:
295 295
296 296 self.dataOut.data = buffer[:, thisProfile, :]
297 297
298 298 shape = [nChannels, nHeis]
299 299
300 300 Si self.getByBlock == True:
301 301
302 302 self.dataOut.data = buffer[:, :, :]
303 303
304 304 shape = [nChannels, nProfiles, nHeis]
305 305
306 306 Variables afectadas:
307 307 self.dataOut
308 308 self.profileIndex
309 309
310 310 Affected:
311 311 self.dataOut
312 312 self.profileIndex
313 313 self.flagTimeBlock
314 314 self.flagIsNewBlock
315 315 """
316 316
317 317 if self.flagNoMoreFiles:
318 318 self.dataOut.flagNoData = True
319 319 print 'Process finished'
320 320 return 0
321 321
322 322 self.flagTimeBlock = 0
323 323 self.flagIsNewBlock = 0
324 324
325 325 if self.__hasNotDataInBuffer():
326 326
327 327 if not( self.readNextBlock() ):
328 328 return 0
329 329
330 330 self.getFirstHeader()
331 331
332 332 if self.datablock == None:
333 333 self.dataOut.flagNoData = True
334 334 return 0
335 335
336 336 if not self.getByBlock:
337 337
338 338 """
339 339 Return profile by profile
340 340
341 341 If nTxs > 1 then one profile is divided by nTxs and number of total
342 342 blocks is increased by nTxs (nProfiles *= nTxs)
343 343 """
344 344 if self.nTxs == 1:
345 345 self.dataOut.flagDataAsBlock = False
346 346 self.dataOut.data = self.datablock[:,self.profileIndex,:]
347 347 self.dataOut.profileIndex = self.profileIndex
348 348
349 349 self.profileIndex += 1
350 350
351 351 else:
352 352 self.dataOut.flagDataAsBlock = False
353 353
354 354 iniHei_ForThisTx = (self.txIndex)*int(self.processingHeaderObj.nHeights/self.nTxs)
355 355 endHei_ForThisTx = (self.txIndex+1)*int(self.processingHeaderObj.nHeights/self.nTxs)
356 356
357 357 # print iniHei_ForThisTx, endHei_ForThisTx
358 358
359 359 self.dataOut.data = self.datablock[:, self.profileIndex, iniHei_ForThisTx:endHei_ForThisTx]
360 360 self.dataOut.profileIndex = self.profileIndex*self.nTxs + self.txIndex
361 361
362 362 self.txIndex += 1
363 363
364 364 if self.txIndex == self.nTxs:
365 365 self.txIndex = 0
366 366 self.profileIndex += 1
367 367
368 368 else:
369 369 """
370 370 Return all block
371 371 """
372 372 self.dataOut.flagDataAsBlock = True
373 373 self.dataOut.data = self.datablock
374 374 self.dataOut.profileIndex = self.processingHeaderObj.profilesPerBlock - 1
375 375
376 376 self.profileIndex = self.processingHeaderObj.profilesPerBlock - 1
377 377
378 378 self.dataOut.flagNoData = False
379 379
380 380 self.getBasicHeader()
381 381
382 382 self.dataOut.realtime = self.online
383 383
384 384 return self.dataOut.data
385 385
386 386 class VoltageWriter(JRODataWriter, Operation):
387 387 """
388 388 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
389 389 de los datos siempre se realiza por bloques.
390 390 """
391 391
392 392 ext = ".r"
393 393
394 394 optchar = "D"
395 395
396 396 shapeBuffer = None
397 397
398 398
399 399 def __init__(self):
400 400 """
401 401 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
402 402
403 403 Affected:
404 404 self.dataOut
405 405
406 406 Return: None
407 407 """
408 408 Operation.__init__(self)
409 409
410 410 self.nTotalBlocks = 0
411 411
412 412 self.profileIndex = 0
413 413
414 414 self.isConfig = False
415 415
416 416 self.fp = None
417 417
418 418 self.flagIsNewFile = 1
419 419
420 420 self.nTotalBlocks = 0
421 421
422 422 self.flagIsNewBlock = 0
423 423
424 424 self.setFile = None
425 425
426 426 self.dtype = None
427 427
428 428 self.path = None
429 429
430 430 self.filename = None
431 431
432 432 self.basicHeaderObj = BasicHeader(LOCALTIME)
433 433
434 434 self.systemHeaderObj = SystemHeader()
435 435
436 436 self.radarControllerHeaderObj = RadarControllerHeader()
437 437
438 438 self.processingHeaderObj = ProcessingHeader()
439 439
440 440 def hasAllDataInBuffer(self):
441 441 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
442 442 return 1
443 443 return 0
444 444
445 445
446 446 def setBlockDimension(self):
447 447 """
448 448 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
449 449
450 450 Affected:
451 451 self.shape_spc_Buffer
452 452 self.shape_cspc_Buffer
453 453 self.shape_dc_Buffer
454 454
455 455 Return: None
456 456 """
457 457 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
458 458 self.processingHeaderObj.nHeights,
459 459 self.systemHeaderObj.nChannels)
460 460
461 461 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
462 462 self.processingHeaderObj.profilesPerBlock,
463 463 self.processingHeaderObj.nHeights),
464 464 dtype=numpy.dtype('complex64'))
465 465
466 466
467 467 def writeBlock(self):
468 468 """
469 469 Escribe el buffer en el file designado
470 470
471 471 Affected:
472 472 self.profileIndex
473 473 self.flagIsNewFile
474 474 self.flagIsNewBlock
475 475 self.nTotalBlocks
476 476 self.blockIndex
477 477
478 478 Return: None
479 479 """
480 480 data = numpy.zeros( self.shapeBuffer, self.dtype )
481 481
482 482 junk = numpy.transpose(self.datablock, (1,2,0))
483 483
484 484 data['real'] = junk.real
485 485 data['imag'] = junk.imag
486 486
487 487 data = data.reshape( (-1) )
488 488
489 489 data.tofile( self.fp )
490 490
491 491 self.datablock.fill(0)
492 492
493 493 self.profileIndex = 0
494 494 self.flagIsNewFile = 0
495 495 self.flagIsNewBlock = 1
496 496
497 497 self.blockIndex += 1
498 498 self.nTotalBlocks += 1
499 499
500 500 def putData(self):
501 501 """
502 502 Setea un bloque de datos y luego los escribe en un file
503 503
504 504 Affected:
505 505 self.flagIsNewBlock
506 506 self.profileIndex
507 507
508 508 Return:
509 509 0 : Si no hay data o no hay mas files que puedan escribirse
510 510 1 : Si se escribio la data de un bloque en un file
511 511 """
512 512 if self.dataOut.flagNoData:
513 513 return 0
514 514
515 515 self.flagIsNewBlock = 0
516 516
517 517 if self.dataOut.flagTimeBlock:
518 518
519 519 self.datablock.fill(0)
520 520 self.profileIndex = 0
521 521 self.setNextFile()
522 522
523 523 if self.profileIndex == 0:
524 524 self.setBasicHeader()
525 525
526 526 self.datablock[:,self.profileIndex,:] = self.dataOut.data
527 527
528 528 self.profileIndex += 1
529 529
530 530 if self.hasAllDataInBuffer():
531 531 #if self.flagIsNewFile:
532 532 self.writeNextBlock()
533 533 # self.setFirstHeader()
534 534
535 535 return 1
536 536
537 537 def __getProcessFlags(self):
538 538
539 539 processFlags = 0
540 540
541 541 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
542 542 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
543 543 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
544 544 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
545 545 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
546 546 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
547 547
548 548 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
549 549
550 550
551 551
552 552 datatypeValueList = [PROCFLAG.DATATYPE_CHAR,
553 553 PROCFLAG.DATATYPE_SHORT,
554 554 PROCFLAG.DATATYPE_LONG,
555 555 PROCFLAG.DATATYPE_INT64,
556 556 PROCFLAG.DATATYPE_FLOAT,
557 557 PROCFLAG.DATATYPE_DOUBLE]
558 558
559 559
560 560 for index in range(len(dtypeList)):
561 561 if self.dataOut.dtype == dtypeList[index]:
562 562 dtypeValue = datatypeValueList[index]
563 563 break
564 564
565 565 processFlags += dtypeValue
566 566
567 567 if self.dataOut.flagDecodeData:
568 568 processFlags += PROCFLAG.DECODE_DATA
569 569
570 570 if self.dataOut.flagDeflipData:
571 571 processFlags += PROCFLAG.DEFLIP_DATA
572 572
573 573 if self.dataOut.code != None:
574 574 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
575 575
576 576 if self.dataOut.nCohInt > 1:
577 577 processFlags += PROCFLAG.COHERENT_INTEGRATION
578 578
579 579 return processFlags
580 580
581 581
582 582 def __getBlockSize(self):
583 583 '''
584 584 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
585 585 '''
586 586
587 587 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
588 588 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
589 589 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
590 590 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
591 591 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
592 592 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
593 593
594 594 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
595 595 datatypeValueList = [1,2,4,8,4,8]
596 596 for index in range(len(dtypeList)):
597 597 if self.dataOut.dtype == dtypeList[index]:
598 598 datatypeValue = datatypeValueList[index]
599 599 break
600 600
601 601 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * datatypeValue * 2)
602 602
603 603 return blocksize
604 604
605 605 def setFirstHeader(self):
606 606
607 607 """
608 608 Obtiene una copia del First Header
609 609
610 610 Affected:
611 611 self.systemHeaderObj
612 612 self.radarControllerHeaderObj
613 613 self.dtype
614 614
615 615 Return:
616 616 None
617 617 """
618 618
619 619 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
620 620 self.systemHeaderObj.nChannels = self.dataOut.nChannels
621 621 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
622 622
623 623 self.setBasicHeader()
624 624
625 625 processingHeaderSize = 40 # bytes
626 626 self.processingHeaderObj.dtype = 0 # Voltage
627 627 self.processingHeaderObj.blockSize = self.__getBlockSize()
628 628 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
629 629 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
630 630 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
631 631 self.processingHeaderObj.processFlags = self.__getProcessFlags()
632 632 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
633 633 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
634 634 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
635 635
636 636 # if self.dataOut.code != None:
637 637 # self.processingHeaderObj.code = self.dataOut.code
638 638 # self.processingHeaderObj.nCode = self.dataOut.nCode
639 639 # self.processingHeaderObj.nBaud = self.dataOut.nBaud
640 640 # codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)
641 641 # processingHeaderSize += codesize
642 642
643 643 if self.processingHeaderObj.nWindows != 0:
644 644 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
645 645 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
646 646 self.processingHeaderObj.nHeights = self.dataOut.nHeights
647 647 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
648 648 processingHeaderSize += 12
649 649
650 650 self.processingHeaderObj.size = processingHeaderSize No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now