##// END OF EJS Templates
schain-jc
RSS
SpectraIO.py:...
Victor Sarmiento -
r86:6f8fdeafc657
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,559 +1,566
1 1 '''
2 2 File: SpectraIO.py
3 3 Created on 20/02/2012
4 4
5 5 @author $Author$
6 6 @version $Id$
7 7 '''
8 8
9 9 import os, sys
10 10 import numpy
11 11 import glob
12 12 import fnmatch
13 13 import time, datetime
14 14
15 15 path = os.path.split(os.getcwd())[0]
16 16 sys.path.append(path)
17 17
18 18 from Model.JROHeader import *
19 19 from Model.Spectra import Spectra
20 20
21 21 from DataIO import JRODataReader
22 22 from DataIO import JRODataWriter
23 23 from DataIO import isNumber
24 24
25 25
26 26 class SpectraReader( JRODataReader ):
27 27 """
28 28 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
29 29 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
30 30 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
31 31
32 32 paresCanalesIguales * alturas * perfiles (Self Spectra)
33 33 paresCanalesDiferentes * alturas * perfiles (Cross Spectra)
34 34 canales * alturas (DC Channels)
35 35
36 36 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
37 37 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
38 38 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
39 39 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
40 40
41 41 Example:
42 42 dpath = "/home/myuser/data"
43 43
44 44 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
45 45
46 46 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
47 47
48 48 readerObj = SpectraReader()
49 49
50 50 readerObj.setup(dpath, startTime, endTime)
51 51
52 52 while(True):
53 53
54 54 readerObj.getData()
55 55
56 56 print readerObj.m_Spectra.data
57 57
58 58 if readerObj.flagNoMoreFiles:
59 59 break
60 60
61 61 """
62 62 m_DataObj = None
63 63
64 64 data_spc = None
65 65 data_cspc = None
66 66 data_dc = None
67 67
68 68 pts2read_SelfSpectra = 0
69 69 pts2read_CrossSpectra = 0
70 70 pts2read_DCchannels = 0
71 71
72 72 nChannels = 0
73 73
74 74 nPairs = 0
75 75
76 pairList = None
76 #pairList = None
77 77
78 channelList = None
78 79
79 80 def __init__(self, m_Spectra=None):
80 81 """
81 82 Inicializador de la clase SpectraReader para la lectura de datos de espectros.
82 83
83 84 Inputs:
84 85 m_Spectra : Objeto de la clase Spectra. Este objeto sera utilizado para
85 86 almacenar un perfil de datos cada vez que se haga un requerimiento
86 87 (getData). El perfil sera obtenido a partir del buffer de datos,
87 88 si el buffer esta vacio se hara un nuevo proceso de lectura de un
88 89 bloque de datos.
89 90 Si este parametro no es pasado se creara uno internamente.
90 91
91 92 Affected:
92 93 self.m_DataObj
93 94
94 95 Return : None
95 96 """
96 97 if m_Spectra == None:
97 98 m_Spectra = Spectra()
98 99
99 100 if not( isinstance(m_Spectra, Spectra) ):
100 101 raise ValueError, "in SpectraReader, m_Spectra must be an Spectra class object"
101 102
102 103 self.m_DataObj = m_Spectra
103 104
104 105 self.data_spc = None
105 106 self.data_cspc = None
106 107 self.data_dc = None
107 108
108 109 self.pts2read_SelfSpectra = 0
109 110 self.pts2read_CrossSpectra = 0
110 self.pts2read_DCchannels = 0
111 self.pts2read_DCs = 0
111 112
112 113 self.nChannels = 0
113 114
114 115 self.nPairs = 0
115 116
116 117 self.ext = ".pdata"
117 118
118 119 self.optchar = "P"
119 120
120 121 ######################
121 122
122 123 self.m_BasicHeader = BasicHeader()
123 124
124 125 self.m_SystemHeader = SystemHeader()
125 126
126 127 self.m_RadarControllerHeader = RadarControllerHeader()
127 128
128 129 self.m_ProcessingHeader = ProcessingHeader()
129 130
130 131 self.online = 0
131 132
132 133 self.fp = None
133 134
134 135 self.fileSizeByHeader = None
135 136
136 137 self.filenameList = []
137 138
138 139 self.filename = None
139 140
140 141 self.fileSize = None
141 142
142 143 self.firstHeaderSize = 0
143 144
144 145 self.basicHeaderSize = 24
145 146
146 147 self.dataType = None
147 148
148 149 self.maxTimeStep = 30
149 150
150 151 self.flagNoMoreFiles = 0
151 152
152 153 self.set = 0
153 154
154 155 self.path = None
155 156
156 157 self.delay = 3 #seconds
157 158
158 159 self.nTries = 3 #quantity tries
159 160
160 161 self.nFiles = 3 #number of files for searching
161 162
162 163 self.nBlocks = 0
163 164
164 165 self.flagIsNewFile = 1
165 166
166 167 self.ippSeconds = 0
167 168
168 169 self.flagResetProcessing = 0
169 170
170 171 self.flagIsNewBlock = 0
171 172
172 173 self.nReadBlocks = 0
173 174
174 175 self.blocksize = 0
175 176
176 pairList = None
177 #pairList = None
178
179 channelList = None
177 180
178 181
179 182 def __hasNotDataInBuffer(self):
180 183 return 1
181 184
182 185
183 186 def getBlockDimension(self):
184 187 """
185 188 Obtiene la cantidad de puntos a leer por cada bloque de datos
186 189
187 190 Affected:
188 191 self.nChannels
189 192 self.nPairs
190 193 self.pts2read_SelfSpectra
191 194 self.pts2read_CrossSpectra
192 195 self.pts2read_DCchannels
193 196 self.blocksize
194 197 self.m_DataObj.nChannels
195 198 self.m_DataObj.nPairs
196 199
197 200 Return:
198 201 None
199 202 """
200 203 self.nChannels = 0
201 204 self.nPairs = 0
205 #self.pairList = []
202 206
203 207 for i in range( 0, self.m_ProcessingHeader.totalSpectra*2, 2 ):
204 208 if self.m_ProcessingHeader.spectraComb[i] == self.m_ProcessingHeader.spectraComb[i+1]:
205 209 self.nChannels = self.nChannels + 1 #par de canales iguales
206 210 else:
207 211 self.nPairs = self.nPairs + 1 #par de canales diferentes
208
212 #self.pairList.append( (self.m_ProcessingHeader.spectraComb[i], self.m_ProcessingHeader.spectraComb[i+1]) )
213
209 214 pts2read = self.m_ProcessingHeader.numHeights * self.m_ProcessingHeader.profilesPerBlock
210 215
211 216 self.pts2read_SelfSpectra = int( self.nChannels * pts2read )
212 217 self.pts2read_CrossSpectra = int( self.nPairs * pts2read )
213 218 self.pts2read_DCchannels = int( self.m_SystemHeader.numChannels * self.m_ProcessingHeader.numHeights )
214 219
215 220 self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
216 221
217 222 self.m_DataObj.nPoints = self.m_ProcessingHeader.profilesPerBlock
218 223 self.m_DataObj.nChannels = self.nChannels
219 224 self.m_DataObj.nPairs = self.nPairs
220 225
221 self.pairList = numpy.arange(self.nPairs)
226 #self.pairList = tuple( self.pairList )
227 self.channelList = numpy.arange( self.nChannels )
222 228
223 229
224 230 def readBlock(self):
225 231 """
226 232 Lee el bloque de datos desde la posicion actual del puntero del archivo
227 233 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
228 234 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
229 235 es seteado a 0
230 236
231 237 Return: None
232 238
233 239 Variables afectadas:
234 240 self.datablockIndex
235 241 self.flagIsNewFile
236 242 self.flagIsNewBlock
237 243 self.nReadBlocks
238 244 self.data_spc
239 245 self.data_cspc
240 246 self.data_dc
241 247
242 248 Exceptions:
243 249 Si un bloque leido no es un bloque valido
244 250 """
245 251 blockOk_flag = False
246 252 fpointer = self.fp.tell()
247 253
248 254 spc = numpy.fromfile( self.fp, self.dataType[0], self.pts2read_SelfSpectra )
249 255 cspc = numpy.fromfile( self.fp, self.dataType, self.pts2read_CrossSpectra )
250 256 dc = numpy.fromfile( self.fp, self.dataType, self.pts2read_DCchannels ) #int(self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels) )
251 257
252 258 if self.online:
253 259 if (spc.size + cspc.size + dc.size) != self.blocksize:
254 260 for nTries in range( self.nTries ):
255 261 print "\tWaiting %0.2f sec for the next block, try %03d ..." % (self.delay, nTries+1)
256 262 time.sleep( self.delay )
257 263 self.fp.seek( fpointer )
258 264 fpointer = self.fp.tell()
259 265
260 266 spc = numpy.fromfile( self.fp, self.dataType[0], self.pts2read_SelfSpectra )
261 267 cspc = numpy.fromfile( self.fp, self.dataType, self.pts2read_CrossSpectra )
262 268 dc = numpy.fromfile( self.fp, self.dataType, self.pts2read_DCchannels ) #int(self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels) )
263 269
264 270 if (spc.size + cspc.size + dc.size) == self.blocksize:
265 271 blockOk_flag = True
266 272 break
267 273
268 274 if not( blockOk_flag ):
269 275 return 0
270 276
271 277 try:
272 278 spc = spc.reshape( (self.nChannels, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D
273 279 if self.nPairs != 0:
274 280 cspc = cspc.reshape( (self.nPairs, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D
275 281 else:
276 282 cspc = None
277 283 dc = dc.reshape( (self.m_SystemHeader.numChannels, self.m_ProcessingHeader.numHeights) ) #transforma a un arreglo 2D
278 284 except:
279 285 print "Data file %s is invalid" % self.filename
280 286 return 0
281 287
282 288 if not( self.m_ProcessingHeader.shif_fft ):
283 289 spc = numpy.roll( spc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
284 290 if cspc != None:
285 291 cspc = numpy.roll( cspc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
286 292
287 293 spc = numpy.transpose( spc, (0,2,1) )
288 294 if cspc != None: cspc = numpy.transpose( cspc, (0,2,1) )
289 295
290 296 self.data_spc = spc
291 297 if cspc != None:
292 298 self.data_cspc = cspc['real'] + cspc['imag']*1j
293 299 else:
294 300 self.data_cspc = None
295 301 self.data_dc = dc['real'] + dc['imag']*1j
296 302
297 303 self.datablockIndex = 0
298 304 self.flagIsNewFile = 0
299 305 self.flagIsNewBlock = 1
300 306
301 307 self.nReadBlocks += 1
302 308 self.nBlocks += 1
303 309
304 310 return 1
305 311
306 312
307 313 def getData(self):
308 314 """
309 315 Copia el buffer de lectura a la clase "Spectra",
310 316 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
311 317 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
312 318
313 319 Return:
314 320 0 : Si no hay mas archivos disponibles
315 321 1 : Si hizo una buena copia del buffer
316 322
317 323 Affected:
318 324 self.m_DataObj
319 325 self.datablockIndex
320 326 self.flagResetProcessing
321 327 self.flagIsNewBlock
322 328 """
323 329
324 330 if self.flagNoMoreFiles: return 0
325 331
326 332 self.flagResetProcessing = 0
327 333 self.flagIsNewBlock = 0
328 334
329 335 if self.__hasNotDataInBuffer():
330 336
331 337 if not( self.readNextBlock() ):
332 338 self.setNextFile()
333 339 return 0
334 340
335 341 self.m_DataObj.m_BasicHeader = self.m_BasicHeader.copy()
336 342 self.m_DataObj.m_ProcessingHeader = self.m_ProcessingHeader.copy()
337 343 self.m_DataObj.m_RadarControllerHeader = self.m_RadarControllerHeader.copy()
338 344 self.m_DataObj.m_SystemHeader = self.m_SystemHeader.copy()
339 345 self.m_DataObj.heightList = self.heightList
340 346 self.m_DataObj.dataType = self.dataType
341 347
342 348 if self.flagNoMoreFiles == 1:
343 349 print 'Process finished'
344 350 return 0
345 351
346 352 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
347 353
348 354 if self.data_dc == None:
349 355 self.m_DataObj.flagNoData = True
350 356 return 0
351 357
352 358 self.m_DataObj.flagNoData = False
353 359 self.m_DataObj.flagResetProcessing = self.flagResetProcessing
354 360
355 361 self.m_DataObj.data_spc = self.data_spc
356 362 self.m_DataObj.data_cspc = self.data_cspc
357 363 self.m_DataObj.data_dc = self.data_dc
358 364
359 365 return 1
360 366
361 367
362 368 class SpectraWriter(JRODataWriter):
363 369
364 370 """
365 371 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
366 372 de los datos siempre se realiza por bloques.
367 373 """
368 374
369 375 m_DataObj = None
370 376
371 377 shape_spc_Buffer = None
372 378 shape_cspc_Buffer = None
373 379 shape_dc_Buffer = None
374 380
375 381 data_spc = None
376 382 data_cspc = None
377 383 data_dc = None
378 384
379 385
380 386 def __init__(self, m_Spectra=None):
381 387 """
382 388 Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
383 389
384 390 Affected:
385 391 self.m_DataObj
386 392 self.m_BasicHeader
387 393 self.m_SystemHeader
388 394 self.m_RadarControllerHeader
389 395 self.m_ProcessingHeader
390 396
391 397 Return: None
392 398 """
393 399 if m_Spectra == None:
394 400 m_Spectra = Spectra()
395 401
396 402 if not( isinstance(m_Spectra, Spectra) ):
397 403 raise ValueError, "in SpectraReader, m_Spectra must be an Spectra class object"
398 404
399 405 self.m_DataObj = m_Spectra
400 406
401 407 self.ext = ".pdata"
402 408
403 409 self.optchar = "P"
404 410
405 411 self.shape_spc_Buffer = None
406 412 self.shape_cspc_Buffer = None
407 413 self.shape_dc_Buffer = None
408 414
409 415 self.data_spc = None
410 416 self.data_cspc = None
411 417 self.data_dc = None
412 418
413 419 ####################################
414 420
415 421 self.fp = None
416 422
417 423 self.blocksCounter = 0
418 424
419 425 self.flagIsNewFile = 1
420 426
421 427 self.nWriteBlocks = 0
422 428
423 429 self.flagIsNewBlock = 0
424 430
425 431 self.flagNoMoreFiles = 0
426 432
427 433 self.setFile = None
428 434
429 435 self.dataType = None
430 436
431 437 self.path = None
432 438
433 439 self.noMoreFiles = 0
434 440
435 441 self.filename = None
436 442
437 443 self.m_BasicHeader= BasicHeader()
438 444
439 445 self.m_SystemHeader = SystemHeader()
440 446
441 447 self.m_RadarControllerHeader = RadarControllerHeader()
442 448
443 449 self.m_ProcessingHeader = ProcessingHeader()
444 450
445 451
446 452 def hasAllDataInBuffer(self):
447 453 return 1
448 454
449 455
450 456 def setBlockDimension(self):
451 457 """
452 458 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
453 459
454 460 Affected:
455 461 self.shape_spc_Buffer
456 462 self.shape_cspc_Buffer
457 463 self.shape_dc_Buffer
458 464
459 465 Return: None
460 466 """
461 467 self.shape_spc_Buffer = (self.m_DataObj.nChannels,
462 468 self.m_ProcessingHeader.numHeights,
463 469 self.m_ProcessingHeader.profilesPerBlock)
464 470
465 471 self.shape_cspc_Buffer = (self.m_DataObj.nPairs,
466 472 self.m_ProcessingHeader.numHeights,
467 473 self.m_ProcessingHeader.profilesPerBlock)
468 474
469 475 self.shape_dc_Buffer = (self.m_SystemHeader.numChannels,
470 476 self.m_ProcessingHeader.numHeights)
471 477
472 478
473 479 def writeBlock(self):
474 480 """
475 481 Escribe el buffer en el file designado
476 482
477 483 Affected:
478 484 self.data_spc
479 485 self.data_cspc
480 486 self.data_dc
481 487 self.flagIsNewFile
482 488 self.flagIsNewBlock
483 489 self.nWriteBlocks
484 490 self.blocksCounter
485 491
486 492 Return: None
487 493 """
488 494
489 495 spc = numpy.transpose( self.data_spc, (0,2,1) )
490 496 if not( self.m_ProcessingHeader.shif_fft ):
491 497 spc = numpy.roll( spc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
492 498 data = spc.reshape((-1))
493 499 data.tofile(self.fp)
494 500
495 501 if self.data_cspc != None:
496 502 data = numpy.zeros( self.shape_cspc_Buffer, self.dataType )
497 503 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
498 504 if not( self.m_ProcessingHeader.shif_fft ):
499 505 cspc = numpy.roll( cspc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
500 506 data['real'] = cspc.real
501 507 data['imag'] = cspc.imag
502 508 data = data.reshape((-1))
503 509 data.tofile(self.fp)
504 510
505 511 data = numpy.zeros( self.shape_dc_Buffer, self.dataType )
506 512 dc = self.data_dc
507 513 data['real'] = dc.real
508 514 data['imag'] = dc.imag
509 515 data = data.reshape((-1))
510 516 data.tofile(self.fp)
511 517
512 518 self.data_spc.fill(0)
513 self.data_cspc.fill(0)
514 519 self.data_dc.fill(0)
520 if self.data_cspc != None:
521 self.data_cspc.fill(0)
515 522
516 523 self.flagIsNewFile = 0
517 524 self.flagIsNewBlock = 1
518 525 self.nWriteBlocks += 1
519 526 self.blocksCounter += 1
520 527
521 528
522 529 def putData(self):
523 530 """
524 531 Setea un bloque de datos y luego los escribe en un file
525 532
526 533 Affected:
527 534 self.data_spc
528 535 self.data_cspc
529 536 self.data_dc
530 537
531 538 Return:
532 539 0 : Si no hay data o no hay mas files que puedan escribirse
533 540 1 : Si se escribio la data de un bloque en un file
534 541 """
535 542 self.flagIsNewBlock = 0
536 543
537 544 if self.m_DataObj.flagNoData:
538 545 return 0
539 546
540 547 if self.m_DataObj.flagResetProcessing:
541 548 self.data_spc.fill(0)
542 549 self.data_cspc.fill(0)
543 550 self.data_dc.fill(0)
544 551 self.setNextFile()
545 552
546 553 self.data_spc = self.m_DataObj.data_spc
547 554 self.data_cspc = self.m_DataObj.data_cspc
548 555 self.data_dc = self.m_DataObj.data_dc
549 556
550 557 # #self.m_ProcessingHeader.dataBlocksPerFile)
551 558 if self.hasAllDataInBuffer():
552 559 self.getHeader()
553 560 self.writeNextBlock()
554 561
555 562 if self.flagNoMoreFiles:
556 563 #print 'Process finished'
557 564 return 0
558 565
559 566 return 1 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now