##// END OF EJS Templates
schain
RSS
SpectraIO.py...
Victor Sarmiento -
r79:bbf69d216d45
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,434 +1,559
1 1 '''
2 2 File: SpectraIO.py
3 3 Created on 20/02/2012
4 4
5 5 @author $Author$
6 6 @version $Id$
7 7 '''
8 8
9 9 import os, sys
10 10 import numpy
11 11 import glob
12 12 import fnmatch
13 13 import time, datetime
14 14
15 15 path = os.path.split(os.getcwd())[0]
16 16 sys.path.append(path)
17 17
18 18 from Model.JROHeader import *
19 19 from Model.Spectra import Spectra
20 20
21 21 from DataIO import JRODataReader
22 22 from DataIO import JRODataWriter
23 23 from DataIO import isNumber
24 24
25 25
26 26 class SpectraReader( JRODataReader ):
27 27 """
28 28 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
29 29 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
30 30 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
31 31
32 32 paresCanalesIguales * alturas * perfiles (Self Spectra)
33 33 paresCanalesDiferentes * alturas * perfiles (Cross Spectra)
34 34 canales * alturas (DC Channels)
35 35
36 36 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
37 37 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
38 38 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
39 39 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
40 40
41 41 Example:
42 42 dpath = "/home/myuser/data"
43 43
44 44 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
45 45
46 46 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
47 47
48 48 readerObj = SpectraReader()
49 49
50 50 readerObj.setup(dpath, startTime, endTime)
51 51
52 52 while(True):
53 53
54 54 readerObj.getData()
55 55
56 56 print readerObj.m_Spectra.data
57 57
58 58 if readerObj.flagNoMoreFiles:
59 59 break
60 60
61 61 """
62 62 m_DataObj = None
63 63
64 64 data_spc = None
65 65 data_cspc = None
66 66 data_dc = None
67 67
68 68 pts2read_SelfSpectra = 0
69 69 pts2read_CrossSpectra = 0
70 70 pts2read_DCchannels = 0
71 71
72 nPairsEqualChannels = 0
72 nChannels = 0
73 73
74 nPairsUnequalChannels = 0
74 nPairs = 0
75 75
76 ext = ".pdata"
77
78 optchar = "P"
76 pairList = None
79 77
80 78
81 79 def __init__(self, m_Spectra=None):
82 80 """
83 81 Inicializador de la clase SpectraReader para la lectura de datos de espectros.
84 82
85 83 Inputs:
86 84 m_Spectra : Objeto de la clase Spectra. Este objeto sera utilizado para
87 85 almacenar un perfil de datos cada vez que se haga un requerimiento
88 86 (getData). El perfil sera obtenido a partir del buffer de datos,
89 87 si el buffer esta vacio se hara un nuevo proceso de lectura de un
90 88 bloque de datos.
91 89 Si este parametro no es pasado se creara uno internamente.
92 90
93 91 Affected:
94 92 self.m_DataObj
95 93
96 94 Return : None
97 95 """
98 96 if m_Spectra == None:
99 97 m_Spectra = Spectra()
100 98
101 99 if not( isinstance(m_Spectra, Spectra) ):
102 100 raise ValueError, "in SpectraReader, m_Spectra must be an Spectra class object"
103 101
104 102 self.m_DataObj = m_Spectra
105 103
104 self.data_spc = None
105 self.data_cspc = None
106 self.data_dc = None
107
108 self.pts2read_SelfSpectra = 0
109 self.pts2read_CrossSpectra = 0
110 self.pts2read_DCchannels = 0
111
112 self.nChannels = 0
113
114 self.nPairs = 0
115
116 self.ext = ".pdata"
117
118 self.optchar = "P"
119
120 ######################
121
122 self.m_BasicHeader = BasicHeader()
123
124 self.m_SystemHeader = SystemHeader()
125
126 self.m_RadarControllerHeader = RadarControllerHeader()
127
128 self.m_ProcessingHeader = ProcessingHeader()
129
130 self.online = 0
131
132 self.fp = None
133
134 self.fileSizeByHeader = None
135
136 self.filenameList = []
137
138 self.filename = None
139
140 self.fileSize = None
141
142 self.firstHeaderSize = 0
143
144 self.basicHeaderSize = 24
145
146 self.dataType = None
147
148 self.maxTimeStep = 30
149
150 self.flagNoMoreFiles = 0
151
152 self.set = 0
153
154 self.path = None
155
156 self.delay = 3 #seconds
157
158 self.nTries = 3 #quantity tries
159
160 self.nFiles = 3 #number of files for searching
161
162 self.nBlocks = 0
163
164 self.flagIsNewFile = 1
165
166 self.ippSeconds = 0
167
168 self.flagResetProcessing = 0
169
170 self.flagIsNewBlock = 0
171
172 self.nReadBlocks = 0
173
174 self.blocksize = 0
175
176 pairList = None
177
106 178
107 179 def __hasNotDataInBuffer(self):
108 180 return 1
109 181
110 182
111 183 def getBlockDimension(self):
112 184 """
113 185 Obtiene la cantidad de puntos a leer por cada bloque de datos
114 186
115 187 Affected:
116 self.nPairsEqualChannels
117 self.nPairsUnequalChannels
188 self.nChannels
189 self.nPairs
118 190 self.pts2read_SelfSpectra
119 191 self.pts2read_CrossSpectra
120 192 self.pts2read_DCchannels
121 193 self.blocksize
122 self.m_DataObj.nPairsEqualChannels
123 self.m_DataObj.nPairsUnequalChannels
194 self.m_DataObj.nChannels
195 self.m_DataObj.nPairs
124 196
125 197 Return:
126 198 None
127 199 """
128 self.nPairsEqualChannels = 0
129 self.nPairsUnequalChannels = 0
200 self.nChannels = 0
201 self.nPairs = 0
130 202
131 203 for i in range( 0, self.m_ProcessingHeader.totalSpectra*2, 2 ):
132 204 if self.m_ProcessingHeader.spectraComb[i] == self.m_ProcessingHeader.spectraComb[i+1]:
133 self.nPairsEqualChannels = self.nPairsEqualChannels + 1 #par de canales iguales
205 self.nChannels = self.nChannels + 1 #par de canales iguales
134 206 else:
135 self.nPairsUnequalChannels = self.nPairsUnequalChannels + 1 #par de canales diferentes
207 self.nPairs = self.nPairs + 1 #par de canales diferentes
136 208
137 209 pts2read = self.m_ProcessingHeader.numHeights * self.m_ProcessingHeader.profilesPerBlock
138 210
139 self.pts2read_SelfSpectra = int( self.nPairsEqualChannels * pts2read )
140 self.pts2read_CrossSpectra = int( self.nPairsUnequalChannels * pts2read )
211 self.pts2read_SelfSpectra = int( self.nChannels * pts2read )
212 self.pts2read_CrossSpectra = int( self.nPairs * pts2read )
141 213 self.pts2read_DCchannels = int( self.m_SystemHeader.numChannels * self.m_ProcessingHeader.numHeights )
142 214
143 215 self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
144 216
145 217 self.m_DataObj.nPoints = self.m_ProcessingHeader.profilesPerBlock
146 self.m_DataObj.nPairsEqualChannels = self.nPairsEqualChannels
147 self.m_DataObj.nPairsUnequalChannels = self.nPairsUnequalChannels
218 self.m_DataObj.nChannels = self.nChannels
219 self.m_DataObj.nPairs = self.nPairs
220
221 self.pairList = numpy.arange(self.nPairs)
148 222
149 223
150 224 def readBlock(self):
151 225 """
152 226 Lee el bloque de datos desde la posicion actual del puntero del archivo
153 227 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
154 228 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
155 229 es seteado a 0
156 230
157 231 Return: None
158 232
159 233 Variables afectadas:
160 234 self.datablockIndex
161 235 self.flagIsNewFile
162 236 self.flagIsNewBlock
163 237 self.nReadBlocks
164 238 self.data_spc
165 239 self.data_cspc
166 240 self.data_dc
167 241
168 242 Exceptions:
169 243 Si un bloque leido no es un bloque valido
170 244 """
171 245 blockOk_flag = False
172 246 fpointer = self.fp.tell()
173 247
174 248 spc = numpy.fromfile( self.fp, self.dataType[0], self.pts2read_SelfSpectra )
175 249 cspc = numpy.fromfile( self.fp, self.dataType, self.pts2read_CrossSpectra )
176 250 dc = numpy.fromfile( self.fp, self.dataType, self.pts2read_DCchannels ) #int(self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels) )
177 251
178 252 if self.online:
179 253 if (spc.size + cspc.size + dc.size) != self.blocksize:
180 254 for nTries in range( self.nTries ):
181 255 print "\tWaiting %0.2f sec for the next block, try %03d ..." % (self.delay, nTries+1)
182 256 time.sleep( self.delay )
183 257 self.fp.seek( fpointer )
184 258 fpointer = self.fp.tell()
185 259
186 260 spc = numpy.fromfile( self.fp, self.dataType[0], self.pts2read_SelfSpectra )
187 261 cspc = numpy.fromfile( self.fp, self.dataType, self.pts2read_CrossSpectra )
188 262 dc = numpy.fromfile( self.fp, self.dataType, self.pts2read_DCchannels ) #int(self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels) )
189 263
190 264 if (spc.size + cspc.size + dc.size) == self.blocksize:
191 265 blockOk_flag = True
192 266 break
193 267
194 268 if not( blockOk_flag ):
195 269 return 0
196 270
197 271 try:
198 spc = spc.reshape( (self.nPairsEqualChannels, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D
199 cspc = cspc.reshape( (self.nPairsUnequalChannels, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D
272 spc = spc.reshape( (self.nChannels, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D
273 if self.nPairs != 0:
274 cspc = cspc.reshape( (self.nPairs, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D
275 else:
276 cspc = None
200 277 dc = dc.reshape( (self.m_SystemHeader.numChannels, self.m_ProcessingHeader.numHeights) ) #transforma a un arreglo 2D
201 278 except:
202 279 print "Data file %s is invalid" % self.filename
203 280 return 0
204 281
205 282 if not( self.m_ProcessingHeader.shif_fft ):
206 283 spc = numpy.roll( spc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
284 if cspc != None:
207 285 cspc = numpy.roll( cspc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
208 286
209 # spc = numpy.transpose( spc, (0,2,1) )
210 # cspc = numpy.transpose( cspc, (0,2,1) )
211 #dc = numpy.transpose(dc, (0,2,1))
287 spc = numpy.transpose( spc, (0,2,1) )
288 if cspc != None: cspc = numpy.transpose( cspc, (0,2,1) )
212 289
213 290 self.data_spc = spc
291 if cspc != None:
214 292 self.data_cspc = cspc['real'] + cspc['imag']*1j
293 else:
294 self.data_cspc = None
215 295 self.data_dc = dc['real'] + dc['imag']*1j
216 296
217 297 self.datablockIndex = 0
218 298 self.flagIsNewFile = 0
219 299 self.flagIsNewBlock = 1
220 300
221 301 self.nReadBlocks += 1
222 302 self.nBlocks += 1
223 303
224 304 return 1
225 305
226 306
227 307 def getData(self):
228 308 """
229 309 Copia el buffer de lectura a la clase "Spectra",
230 310 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
231 311 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
232 312
233 313 Return:
234 314 0 : Si no hay mas archivos disponibles
235 315 1 : Si hizo una buena copia del buffer
236 316
237 317 Affected:
238 318 self.m_DataObj
239 319 self.datablockIndex
240 320 self.flagResetProcessing
241 321 self.flagIsNewBlock
242 322 """
243 323
244 324 if self.flagNoMoreFiles: return 0
245 325
246 326 self.flagResetProcessing = 0
247 327 self.flagIsNewBlock = 0
248 328
249 329 if self.__hasNotDataInBuffer():
250 330
251 331 if not( self.readNextBlock() ):
252 332 self.setNextFile()
253 333 return 0
254 334
255 335 self.m_DataObj.m_BasicHeader = self.m_BasicHeader.copy()
256 336 self.m_DataObj.m_ProcessingHeader = self.m_ProcessingHeader.copy()
257 337 self.m_DataObj.m_RadarControllerHeader = self.m_RadarControllerHeader.copy()
258 338 self.m_DataObj.m_SystemHeader = self.m_SystemHeader.copy()
259 self.m_DataObj.heights = self.heights
339 self.m_DataObj.heightList = self.heightList
260 340 self.m_DataObj.dataType = self.dataType
261 341
262 342 if self.flagNoMoreFiles == 1:
263 343 print 'Process finished'
264 344 return 0
265 345
266 346 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
267 347
268 348 if self.data_dc == None:
269 349 self.m_DataObj.flagNoData = True
270 350 return 0
271 351
272 352 self.m_DataObj.flagNoData = False
273 353 self.m_DataObj.flagResetProcessing = self.flagResetProcessing
274 354
275 355 self.m_DataObj.data_spc = self.data_spc
276 356 self.m_DataObj.data_cspc = self.data_cspc
277 357 self.m_DataObj.data_dc = self.data_dc
278 358
279 359 return 1
280 360
281 361
282 362 class SpectraWriter(JRODataWriter):
283 363
284 364 """
285 365 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
286 366 de los datos siempre se realiza por bloques.
287 367 """
288 368
289 369 m_DataObj = None
290 370
291 ext = ".pdata"
292
293 optchar = "P"
294
295 371 shape_spc_Buffer = None
296 372 shape_cspc_Buffer = None
297 373 shape_dc_Buffer = None
298 374
375 data_spc = None
376 data_cspc = None
377 data_dc = None
378
299 379
300 380 def __init__(self, m_Spectra=None):
301 381 """
302 382 Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
303 383
304 384 Affected:
305 385 self.m_DataObj
306 386 self.m_BasicHeader
307 387 self.m_SystemHeader
308 388 self.m_RadarControllerHeader
309 389 self.m_ProcessingHeader
310 390
311 391 Return: None
312 392 """
313 393 if m_Spectra == None:
314 394 m_Spectra = Spectra()
315 395
316 396 if not( isinstance(m_Spectra, Spectra) ):
317 397 raise ValueError, "in SpectraReader, m_Spectra must be an Spectra class object"
318 398
319 399 self.m_DataObj = m_Spectra
320 400
401 self.ext = ".pdata"
402
403 self.optchar = "P"
404
405 self.shape_spc_Buffer = None
406 self.shape_cspc_Buffer = None
407 self.shape_dc_Buffer = None
408
409 self.data_spc = None
410 self.data_cspc = None
411 self.data_dc = None
412
413 ####################################
414
415 self.fp = None
416
417 self.blocksCounter = 0
418
419 self.flagIsNewFile = 1
420
421 self.nWriteBlocks = 0
422
423 self.flagIsNewBlock = 0
424
425 self.flagNoMoreFiles = 0
426
427 self.setFile = None
428
429 self.dataType = None
430
431 self.path = None
432
433 self.noMoreFiles = 0
434
435 self.filename = None
436
437 self.m_BasicHeader= BasicHeader()
438
439 self.m_SystemHeader = SystemHeader()
440
441 self.m_RadarControllerHeader = RadarControllerHeader()
442
443 self.m_ProcessingHeader = ProcessingHeader()
444
321 445
322 446 def hasAllDataInBuffer(self):
323 447 return 1
324 448
325 449
326 450 def setBlockDimension(self):
327 451 """
328 452 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
329 453
330 454 Affected:
331 455 self.shape_spc_Buffer
332 456 self.shape_cspc_Buffer
333 457 self.shape_dc_Buffer
334 458
335 459 Return: None
336 460 """
337 self.shape_spc_Buffer = (self.m_DataObj.nPairsEqualChannels,
461 self.shape_spc_Buffer = (self.m_DataObj.nChannels,
338 462 self.m_ProcessingHeader.numHeights,
339 463 self.m_ProcessingHeader.profilesPerBlock)
340 464
341 self.shape_cspc_Buffer = (self.m_DataObj.nPairsUnequalChannels,
465 self.shape_cspc_Buffer = (self.m_DataObj.nPairs,
342 466 self.m_ProcessingHeader.numHeights,
343 467 self.m_ProcessingHeader.profilesPerBlock)
344 468
345 469 self.shape_dc_Buffer = (self.m_SystemHeader.numChannels,
346 470 self.m_ProcessingHeader.numHeights)
347 471
348 472
349 473 def writeBlock(self):
350 474 """
351 475 Escribe el buffer en el file designado
352 476
353 477 Affected:
354 478 self.data_spc
355 479 self.data_cspc
356 480 self.data_dc
357 481 self.flagIsNewFile
358 482 self.flagIsNewBlock
359 483 self.nWriteBlocks
360 484 self.blocksCounter
361 485
362 486 Return: None
363 487 """
364 spc = self.data_spc
365 # spc = numpy.transpose( self.data_spc, (0,2,1) )
488
489 spc = numpy.transpose( self.data_spc, (0,2,1) )
366 490 if not( self.m_ProcessingHeader.shif_fft ):
367 491 spc = numpy.roll( spc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
368 492 data = spc.reshape((-1))
369 493 data.tofile(self.fp)
370 494
495 if self.data_cspc != None:
371 496 data = numpy.zeros( self.shape_cspc_Buffer, self.dataType )
372 cspc = self.data_cspc
373 # cspc = numpy.transpose( self.data_cspc, (0,2,1) )
497 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
374 498 if not( self.m_ProcessingHeader.shif_fft ):
375 499 cspc = numpy.roll( cspc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
500 data['real'] = cspc.real
376 501 data['imag'] = cspc.imag
377 502 data = data.reshape((-1))
378 503 data.tofile(self.fp)
379 504
380 505 data = numpy.zeros( self.shape_dc_Buffer, self.dataType )
381 506 dc = self.data_dc
382 507 data['real'] = dc.real
383 508 data['imag'] = dc.imag
384 509 data = data.reshape((-1))
385 510 data.tofile(self.fp)
386 511
387 512 self.data_spc.fill(0)
388 513 self.data_cspc.fill(0)
389 514 self.data_dc.fill(0)
390 515
391 516 self.flagIsNewFile = 0
392 517 self.flagIsNewBlock = 1
393 518 self.nWriteBlocks += 1
394 519 self.blocksCounter += 1
395 520
396 521
397 522 def putData(self):
398 523 """
399 524 Setea un bloque de datos y luego los escribe en un file
400 525
401 526 Affected:
402 527 self.data_spc
403 528 self.data_cspc
404 529 self.data_dc
405 530
406 531 Return:
407 532 0 : Si no hay data o no hay mas files que puedan escribirse
408 533 1 : Si se escribio la data de un bloque en un file
409 534 """
410 535 self.flagIsNewBlock = 0
411 536
412 537 if self.m_DataObj.flagNoData:
413 538 return 0
414 539
415 540 if self.m_DataObj.flagResetProcessing:
416 541 self.data_spc.fill(0)
417 542 self.data_cspc.fill(0)
418 543 self.data_dc.fill(0)
419 544 self.setNextFile()
420 545
421 546 self.data_spc = self.m_DataObj.data_spc
422 547 self.data_cspc = self.m_DataObj.data_cspc
423 548 self.data_dc = self.m_DataObj.data_dc
424 549
425 550 # #self.m_ProcessingHeader.dataBlocksPerFile)
426 if True:
551 if self.hasAllDataInBuffer():
427 552 self.getHeader()
428 553 self.writeNextBlock()
429 554
430 555 if self.flagNoMoreFiles:
431 556 #print 'Process finished'
432 557 return 0
433 558
434 559 return 1 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now