##// END OF EJS Templates
schain
RSS
Optimización del modulo de escritura de voltajes.
Miguel Valdez -
r23:7fedb37b4f03
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,929 +1,945
1 1 '''
2 2 Created on 23/01/2012
3 3
4 4 @author $Author$
5 5 @version $Id$
6 6 '''
7 7
8 8 import os, sys
9 9 import numpy
10 10 import glob
11 11 import fnmatch
12 12 import time, datetime
13 13
14 14 path = os.path.split(os.getcwd())[0]
15 15 sys.path.append(path)
16 16
17 17 from IO.HeaderIO import *
18 18 from IO.DataIO import DataReader
19 19 from IO.DataIO import DataWriter
20 20
21 21 from Model.Voltage import Voltage
22 22
23 23 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
24 24 """
25 25 Esta funcion determina si un archivo de datos en formato Jicamarca(.r) se encuentra
26 26 o no dentro del rango de fecha especificado.
27 27
28 28 Inputs:
29 29 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
30 30
31 31 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
32 32 segundos contados desde 01/01/1970.
33 33 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
34 34 segundos contados desde 01/01/1970.
35 35
36 36 Return:
37 37 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
38 38 fecha especificado, de lo contrario retorna False.
39 39
40 40 Excepciones:
41 41 Si el archivo no existe o no puede ser abierto
42 42 Si la cabecera no puede ser leida.
43 43
44 44 """
45 45 m_BasicHeader = BasicHeader()
46 46
47 47 try:
48 48 fp = open(filename,'rb')
49 49 except:
50 50 raise IOError, "The file %s can't be opened" %(filename)
51 51
52 52 if not(m_BasicHeader.read(fp)):
53 53 raise IOError, "The file %s has not a valid header" %(filename)
54 54
55 55 fp.close()
56 56
57 57 if not ((startUTSeconds <= m_BasicHeader.utc) and (endUTSeconds >= m_BasicHeader.utc)):
58 58 return 0
59 59
60 60 return 1
61 61
62 62 class VoltageReader(DataReader):
63 63 """
64 64 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
65 65 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
66 66 perfiles*alturas*canales) son almacenados en la variable "buffer".
67 67
68 68 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
69 69 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
70 70 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
71 71 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
72 72
73 73 Example:
74 74
75 75 dpath = "/home/myuser/data"
76 76
77 77 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
78 78
79 79 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
80 80
81 81 readerObj = VoltageReader()
82 82
83 83 readerObj.setup(dpath, startTime, endTime)
84 84
85 85 while(True):
86 86
87 87 readerObj.getData()
88 88
89 89 print readerObj.m_Voltage.data
90 90
91 91 if readerObj.noMoreFiles:
92 92 break
93 93
94 94 """
95 95
96 96 #speed of light
97 97 __c = 3E8
98 98
99 99 def __init__(self, m_Voltage = None):
100 100 """
101 101 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
102 102
103 103 Input:
104 104 m_Voltage : Objeto de la clase Voltage. Este objeto sera utilizado para
105 105 almacenar un perfil de datos cada vez que se haga un requerimiento
106 106 (getData). El perfil sera obtenido a partir del buffer de datos,
107 107 si el buffer esta vacio se hara un nuevo proceso de lectura de un
108 108 bloque de datos.
109 109 Si este parametro no es pasado se creara uno internamente.
110 110
111 111 Variables afectadas:
112 112 self.m_Voltage
113 113 self.m_BasicHeader
114 114 self.m_SystemHeader
115 115 self.m_RadarControllerHeader
116 116 self.m_ProcessingHeader
117 117
118 118
119 119 Return:
120 120 Void
121 121
122 122 """
123 123 if m_Voltage == None:
124 124 m_Voltage = Voltage()
125 125
126 126 if not(isinstance(m_Voltage, Voltage)):
127 127 raise ValueError, "in VoltageReader, m_Voltage must be an Voltage class object"
128 128
129 129 self.m_Voltage = m_Voltage
130 130
131 131 self.m_BasicHeader = BasicHeader()
132 132
133 133 self.m_SystemHeader = SystemHeader()
134 134
135 135 self.m_RadarControllerHeader = RadarControllerHeader()
136 136
137 137 self.m_ProcessingHeader = ProcessingHeader()
138 138
139 139 self.__fp = None
140 140
141 141 self.__idFile = None
142 142
143 143 self.__startDateTime = None
144 144
145 145 self.__endDateTime = None
146 146
147 147 self.__dataType = None
148 148
149 149 self.__fileSizeByHeader = 0
150 150
151 151 self.__pathList = []
152 152
153 153 self.filenameList = []
154 154
155 155 self.__lastUTTime = 0
156 156
157 157 self.__maxTimeStep = 30
158 158
159 159 self.__flagIsNewFile = 0
160 160
161 161 self.__ippSeconds = 0
162 162
163 163 self.flagResetProcessing = 0
164 164
165 165 self.flagIsNewBlock = 0
166 166
167 167 self.noMoreFiles = 0
168 168
169 169 self.nReadBlocks = 0
170 170
171 171 self.online = 0
172 172
173 173 self.filename = None
174 174
175 175 self.fileSize = None
176 176
177 177 self.firstHeaderSize = 0
178 178
179 179 self.basicHeaderSize = 24
180 180
181 181 self.idProfile = 0
182 182
183 self.__buffer = 0
183 self.__buffer = None
184 184
185 185 self.__buffer_id = 9999
186 186
187 187 def __rdSystemHeader(self,fp=None):
188 188 if fp == None:
189 189 fp = self.__fp
190 190
191 191 self.m_SystemHeader.read(fp)
192 192
193 193 def __rdRadarControllerHeader(self,fp=None):
194 194 if fp == None:
195 195 fp = self.__fp
196 196
197 197 self.m_RadarControllerHeader.read(fp)
198 198
199 199 def __rdProcessingHeader(self,fp=None):
200 200 if fp == None:
201 201 fp = self.__fp
202 202
203 203 self.m_ProcessingHeader.read(fp)
204 204
205 205 def __rdBasicHeader(self, fp=None):
206 206
207 207 if fp == None:
208 208 fp = self.__fp
209 209
210 210 self.m_BasicHeader.read(fp)
211 211
212 212 def __readFirstHeader(self):
213 213
214 214 self.__rdBasicHeader()
215 215 self.__rdSystemHeader()
216 216 self.__rdRadarControllerHeader()
217 217 self.__rdProcessingHeader()
218 218 self.firstHeaderSize = self.m_BasicHeader.size
219 219
220 220 data_type=int(numpy.log2((self.m_ProcessingHeader.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
221 221 if data_type == 0:
222 222 tmp = numpy.dtype([('real','<i1'),('imag','<i1')])
223 223
224 224 elif data_type == 1:
225 225 tmp = numpy.dtype([('real','<i2'),('imag','<i2')])
226 226
227 227 elif data_type == 2:
228 228 tmp = numpy.dtype([('real','<i4'),('imag','<i4')])
229 229
230 230 elif data_type == 3:
231 231 tmp = numpy.dtype([('real','<i8'),('imag','<i8')])
232 232
233 233 elif data_type == 4:
234 234 tmp = numpy.dtype([('real','<f4'),('imag','<f4')])
235 235
236 236 elif data_type == 5:
237 237 tmp = numpy.dtype([('real','<f8'),('imag','<f8')])
238 238
239 239 else:
240 240 raise ValueError, 'Data type was not defined'
241 241
242 242 xi = self.m_ProcessingHeader.firstHeight
243 243 step = self.m_ProcessingHeader.deltaHeight
244 244 xf = xi + self.m_ProcessingHeader.numHeights*step
245 245
246 246 self.__heights = numpy.arange(xi, xf, step)
247 247 self.__dataType = tmp
248 248 self.__fileSizeByHeader = self.m_ProcessingHeader.dataBlocksPerFile * self.m_ProcessingHeader.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.m_ProcessingHeader.dataBlocksPerFile - 1)
249 249 self.__ippSeconds = 2*1000*self.m_RadarControllerHeader.ipp/self.__c
250 250
251 251 def __setNextFileOnline(self):
252 252 return 0
253 253
254 254 def __setNextFileOffline(self):
255 255
256 256 idFile = self.__idFile
257 257 while(True):
258 258
259 259 idFile += 1
260 260
261 261 if not(idFile < len(self.filenameList)):
262 262 self.noMoreFiles = 1
263 263 return 0
264 264
265 265 filename = self.filenameList[idFile]
266 266 fileSize = os.path.getsize(filename)
267 267
268 268 try:
269 269 fp = open(filename,'rb')
270 270 except:
271 271 raise IOError, "The file %s can't be opened" %filename
272 272
273 273 currentSize = fileSize - fp.tell()
274 274 neededSize = self.m_ProcessingHeader.blockSize + self.firstHeaderSize
275 275
276 276 if (currentSize < neededSize):
277 277 print "Skipping the file %s due to it hasn't enough data" %filename
278 278 continue
279 279
280 280 break
281 281
282 282 self.__flagIsNewFile = 1
283 283 self.__idFile = idFile
284 284 self.filename = filename
285 285 self.fileSize = fileSize
286 286 self.__fp = fp
287 287
288 288 print 'Setting the file: %s'%self.filename
289 289
290 290 return 1
291 291
292 292 def __setNextFile(self):
293 293
294 294 if self.online:
295 295 newFile = self.__setNextFileOnline()
296 296 else:
297 297 newFile = self.__setNextFileOffline()
298 298
299 299 if not(newFile):
300 300 return 0
301 301
302 302 self.__readFirstHeader()
303 303
304 304 return 1
305 305
306 306 def __setNewBlock(self):
307 307
308 308 if self.__fp == None:
309 309 return 0
310 310
311 311 if self.__flagIsNewFile:
312 312 return 1
313 313
314 314 currentSize = self.fileSize - self.__fp.tell()
315 315 neededSize = self.m_ProcessingHeader.blockSize + self.basicHeaderSize
316 316
317 317 #If there is enough data setting new data block
318 318 if (currentSize >= neededSize):
319 319 self.__rdBasicHeader()
320 320 return 1
321 321
322 322 #Setting new file
323 323 if not(self.__setNextFile()):
324 324 return 0
325 325
326 326 deltaTime = self.m_BasicHeader.utc - self.__lastUTTime # check this
327 327
328 328 self.flagResetProcessing = 0
329 329
330 330 if deltaTime > self.__maxTimeStep:
331 331 self.flagResetProcessing = 1
332 332 self.nReadBlocks = 0
333 333
334 334 return 1
335 335
336 336 def __readBlock(self):
337 337 """
338 338 __readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
339 339 (self.__fp) y actualiza todos los parametros relacionados al bloque de datos
340 340 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
341 341 es seteado a 0
342 342
343 343
344 344 Inputs:
345 345 None
346 346
347 347 Return:
348 348 None
349 349
350 350 Variables afectadas:
351 351
352 352 self.__buffer_id
353 353
354 354 self.__buffer
355 355
356 356 self.__flagIsNewFile
357 357
358 358 self.idProfile
359 359
360 360 self.flagIsNewBlock
361 361
362 362 self.nReadBlocks
363 363
364 364 """
365 365
366 366 pts2read = self.m_ProcessingHeader.profilesPerBlock*self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels
367 367
368 368 junk = numpy.fromfile(self.__fp, self.__dataType, pts2read)
369 369
370 370 junk = junk.reshape((self.m_ProcessingHeader.profilesPerBlock, self.m_ProcessingHeader.numHeights, self.m_SystemHeader.numChannels))
371 371
372 372 data = junk['real'] + junk['imag']*1j
373 373
374 374 self.__buffer_id = 0
375 375
376 376 self.__buffer = data
377 377
378 378 self.__flagIsNewFile = 0
379 379
380 380 self.idProfile = 0
381 381
382 382 self.flagIsNewBlock = 1
383 383
384 384 self.nReadBlocks += 1
385 385
386 386 def __hasNotDataInBuffer(self):
387 387 if self.__buffer_id >= self.m_ProcessingHeader.profilesPerBlock:
388 388 return 1
389 389
390 390 return 0
391 391
392 392 def __searchFiles(self, path, startDateTime, endDateTime, set=None, expLabel = "", ext = ".r"):
393 393 """
394 394 __searchFiles realiza una busqueda de los archivos que coincidan con los parametros
395 395 especificados y se encuentren ubicados en el path indicado. Para realizar una busqueda
396 396 correcta la estructura de directorios debe ser la siguiente:
397 397
398 398 ...path/D[yyyy][ddd]/expLabel/D[yyyy][ddd][sss].ext
399 399
400 400 [yyyy]: anio
401 401 [ddd] : dia del anio
402 402 [sss] : set del archivo
403 403
404 404 Inputs:
405 405 path : Directorio de datos donde se realizara la busqueda. Todos los
406 406 ficheros que concidan con el criterio de busqueda seran
407 407 almacenados en una lista y luego retornados.
408 408 startDateTime : Fecha inicial. Rechaza todos los archivos donde
409 409 file end time < startDateTime (obejto datetime.datetime)
410 410
411 411 endDateTime : Fecha final. Rechaza todos los archivos donde
412 412 file start time > endDateTime (obejto datetime.datetime)
413 413
414 414 set : Set del primer archivo a leer. Por defecto None
415 415
416 416 expLabel : Nombre del subdirectorio de datos. Por defecto ""
417 417
418 418 ext : Extension de los archivos a leer. Por defecto .r
419 419
420 420 Return:
421 421
422 422 (pathList, filenameList)
423 423
424 424 pathList : Lista de directorios donde se encontraron archivos dentro
425 425 de los parametros especificados
426 426 filenameList : Lista de archivos (ruta completa) que coincidieron con los
427 427 parametros especificados.
428 428
429 429 Variables afectadas:
430 430
431 431 self.filenameList: Lista de archivos (ruta completa) que la clase utiliza
432 432 como fuente para leer los bloque de datos, si se termina
433 433 de leer todos los bloques de datos de un determinado
434 434 archivo se pasa al siguiente archivo de la lista.
435 435
436 436 Excepciones:
437 437
438 438 """
439 439
440 440 print "Searching files ..."
441 441
442 442 dirList = []
443 443 for thisPath in os.listdir(path):
444 444 if os.path.isdir(os.path.join(path,thisPath)):
445 445 dirList.append(thisPath)
446 446
447 447 pathList = []
448 448
449 449 thisDateTime = startDateTime
450 450
451 451 while(thisDateTime <= endDateTime):
452 452 year = thisDateTime.timetuple().tm_year
453 453 doy = thisDateTime.timetuple().tm_yday
454 454
455 455 match = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy))
456 456 if len(match) == 0:
457 457 thisDateTime += datetime.timedelta(1)
458 458 continue
459 459
460 460 pathList.append(os.path.join(path,match[0],expLabel))
461 461 thisDateTime += datetime.timedelta(1)
462 462
463 463 startUtSeconds = time.mktime(startDateTime.timetuple())
464 464 endUtSeconds = time.mktime(endDateTime.timetuple())
465 465
466 466 filenameList = []
467 467 for thisPath in pathList:
468 468 fileList = glob.glob1(thisPath, "*%s" %ext)
469 469 fileList.sort()
470 470 for file in fileList:
471 471 filename = os.path.join(thisPath,file)
472 472 if isThisFileinRange(filename, startUtSeconds, endUtSeconds):
473 473 filenameList.append(filename)
474 474
475 475 self.filenameList = filenameList
476 476
477 477 return pathList, filenameList
478 478
479 479 def setup(self, path, startDateTime, endDateTime=None, set=None, expLabel = "", ext = ".r", online = 0):
480 480 """
481 481 setup configura los parametros de lectura de la clase VoltageReader.
482 482
483 483 Si el modo de lectura es offline, primero se realiza una busqueda de todos los archivos
484 484 que coincidan con los parametros especificados; esta lista de archivos son almacenados en
485 485 self.filenameList.
486 486
487 487 Input:
488 488 path : Directorios donde se ubican los datos a leer. Dentro de este
489 489 directorio deberia de estar subdirectorios de la forma:
490 490
491 491 path/D[yyyy][ddd]/expLabel/P[yyyy][ddd][sss][ext]
492 492
493 493 startDateTime : Fecha inicial. Rechaza todos los archivos donde
494 494 file end time < startDatetime (obejto datetime.datetime)
495 495
496 496 endDateTime : Fecha final. Si no es None, rechaza todos los archivos donde
497 497 file end time < startDatetime (obejto datetime.datetime)
498 498
499 499 set : Set del primer archivo a leer. Por defecto None
500 500
501 501 expLabel : Nombre del subdirectorio de datos. Por defecto ""
502 502
503 503 ext : Extension de los archivos a leer. Por defecto .r
504 504
505 505 online :
506 506
507 507 Return:
508 508
509 509 Affected:
510 510
511 511 Excepciones:
512 512
513 513 Example:
514 514
515 515 """
516 516
517 517 if online == 0:
518 518 pathList, filenameList = self.__searchFiles(path, startDateTime, endDateTime, set, expLabel, ext)
519 519
520 520 self.__idFile = -1
521 521
522 522 if not(self.__setNextFile()):
523 523 print "No files in range: %s - %s" %(startDateTime.ctime(), endDateTime.ctime())
524 524 return 0
525 525
526 526 self.startUTCSeconds = time.mktime(startDateTime.timetuple())
527 527 self.endUTCSeconds = time.mktime(endDateTime.timetuple())
528 528
529 529 self.startYear = startDateTime.timetuple().tm_year
530 530 self.endYear = endDateTime.timetuple().tm_year
531 531
532 532 self.startDoy = startDateTime.timetuple().tm_yday
533 533 self.endDoy = endDateTime.timetuple().tm_yday
534 534
535 535 self.m_Voltage.m_BasicHeader = self.m_BasicHeader.copy()
536 536 self.m_Voltage.m_ProcessingHeader = self.m_ProcessingHeader.copy()
537 537 self.m_Voltage.m_RadarControllerHeader = self.m_RadarControllerHeader.copy()
538 538 self.m_Voltage.m_SystemHeader = self.m_SystemHeader.copy()
539 539 self.m_Voltage.dataType = self.__dataType
540 540
541 541 self.__pathList = pathList
542 542 self.filenameList = filenameList
543 543 self.online = online
544 544
545 return 1
546
545 547 def readNextBlock(self):
546 548 """
547 549 readNextBlock establece un nuevo bloque de datos a leer y los lee, si es que no existiese
548 550 mas bloques disponibles en el archivo actual salta al siguiente.
549 551
550 552 """
551 553
552 554 if not(self.__setNewBlock()):
553 555 return 0
554 556
555 557 self.__readBlock()
556 558
557 559 self.__lastUTTime = self.m_BasicHeader.utc
558 560
559 561 return 1
560 562
561 563 def getData(self):
562 564 """
563 565 getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"
564 566 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
565 567 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
566 568
567 569 Ademas incrementa el contador del buffer en 1.
568 570
569 571 Inputs:
570 572 None
571 573
572 574 Return:
573 575 data : retorna un perfil de voltages (alturas * canales) copiados desde el
574 576 buffer. Si no hay mas archivos a leer retorna None.
575 577
576 578 Variables afectadas:
577 579 self.m_Voltage
578 580 self.__buffer_id
579 581 self.idProfile
580 582
581 583 Excepciones:
582 584
583 585 """
584 586 self.flagResetProcessing = 0
585 587 self.flagIsNewBlock = 0
586 588
587 if self.__hasNotDataInBuffer():
588 self.readNextBlock()
589
589 if self.__hasNotDataInBuffer():
590 self.readNextBlock()
591
592 self.m_Voltage.m_BasicHeader = self.m_BasicHeader.copy()
593 self.m_Voltage.m_ProcessingHeader = self.m_ProcessingHeader.copy()
594 self.m_Voltage.m_RadarControllerHeader = self.m_RadarControllerHeader.copy()
595 self.m_Voltage.m_SystemHeader = self.m_SystemHeader.copy()
596
597 self.m_Voltage.heights = self.__heights
598 self.m_Voltage.dataType = self.__dataType
599
590 600 if self.noMoreFiles == 1:
591 601 print 'Process finished'
592 602 return None
593 603
594 604 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
595 data = self.__buffer[self.__buffer_id,:,:]
596 605
597 606 time = self.m_BasicHeader.utc + self.__buffer_id*self.__ippSeconds
598
599 self.m_Voltage.m_BasicHeader = self.m_BasicHeader.copy()
600 self.m_Voltage.m_ProcessingHeader = self.m_ProcessingHeader.copy()
601 self.m_Voltage.m_RadarControllerHeader = self.m_RadarControllerHeader.copy()
602 self.m_Voltage.m_SystemHeader = self.m_SystemHeader.copy()
603 self.m_Voltage.m_BasicHeader.utc = time
604 self.m_Voltage.data = data
607 self.m_Voltage.m_BasicHeader.utc = time
608 self.m_Voltage.data = self.__buffer[self.__buffer_id,:,:]
605 609 self.m_Voltage.flagNoData = False
606 610 self.m_Voltage.flagResetProcessing = self.flagResetProcessing
607 self.m_Voltage.heights = self.__heights
611
608 612 self.m_Voltage.idProfile = self.idProfile
609 self.m_Voltage.dataType = self.__dataType
613
610 614
611 615 self.__buffer_id += 1
612 616 self.idProfile += 1
613 617
614 618 #call setData - to Data Object
615 619
616 return data
620 return 1
617 621
618 622 class VoltageWriter(DataWriter):
619 623 __configHeaderFile = 'wrSetHeadet.txt'
620 624
621 625 def __init__(self, m_Voltage = None):
622 626
623 627 if m_Voltage == None:
624 628 m_Voltage = Voltage()
625 629
626 630 self.m_Voltage = m_Voltage
627 631
628 632 self.__path = None
629 633
630 634 self.__fp = None
631 635
632 636 self.__format = None
633 637
634 638 self.__blocksCounter = 0
635 639
636 640 self.__setFile = None
637 641
638 642 self.__flagIsNewFile = 1
639 643
640 self.__buffer = 0
644 self.__buffer = None
641 645
642 646 self.__buffer_id = 0
643 647
644 648 self.__dataType = None
645 649
646 650 self.__ext = None
647 651
652 self.__shapeBuffer = None
653
648 654 self.nWriteBlocks = 0
649 655
650 656 self.flagIsNewBlock = 0
651 657
652 658 self.noMoreFiles = 0
653 659
654 660 self.filename = None
655 661
656 662 self.m_BasicHeader= BasicHeader()
657 663
658 664 self.m_SystemHeader = SystemHeader()
659 665
660 666 self.m_RadarControllerHeader = RadarControllerHeader()
661 667
662 668 self.m_ProcessingHeader = ProcessingHeader()
663 669
664 670
665 671 def __writeBasicHeader(self, fp=None):
666 672 if fp == None:
667 673 fp = self.__fp
668 674
669 675 self.m_BasicHeader.write(fp)
670 676
671 677 def __wrSystemHeader(self,fp=None):
672 678 if fp == None:
673 679 fp = self.__fp
674 680
675 681 self.m_SystemHeader.write(fp)
676 682
677 683 def __wrRadarControllerHeader(self,fp=None):
678 684 if fp == None:
679 685 fp = self.__fp
680 686
681 687 self.m_RadarControllerHeader.write(fp)
682 688
683 689 def __wrProcessingHeader(self,fp=None):
684 690 if fp == None:
685 691 fp = self.__fp
686 692
687 693 self.m_ProcessingHeader.write(fp)
688 694
689 695 def __writeFirstHeader(self):
690 696 self.__writeBasicHeader()
691 697 self.__wrSystemHeader()
692 698 self.__wrRadarControllerHeader()
693 699 self.__wrProcessingHeader()
694 700 self.__dataType = self.m_Voltage.dataType
695 701
696 702
697 703 def __setNextFile(self):
698 704
699 705 setFile = self.__setFile
700 706 ext = self.__ext
701 707 path = self.__path
702 708
703 709 setFile += 1
704 710
705 711 if self.__fp != None:
706 712 self.__fp.close()
707 713
708 714 timeTuple = time.localtime(self.m_Voltage.m_BasicHeader.utc) # utc from m_Voltage
709 715 file = 'D%4.4d%3.3d%3.3d%s' % (timeTuple.tm_year,timeTuple.tm_yday,setFile,ext)
710 716 subfolder = 'D%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
711 717 tmp = os.path.join(path,subfolder)
712 718 if not(os.path.exists(tmp)):
713 719 os.mkdir(tmp)
714 720
715 721 filename = os.path.join(path,subfolder,file)
716 722 fp = open(filename,'wb')
717 723
718 724 self.__blocksCounter = 0
719 725
720 726 #guardando atributos
721 727 self.filename = filename
722 728 self.__subfolder = subfolder
723 729 self.__fp = fp
724 730 self.__setFile = setFile
725 731 self.__flagIsNewFile = 1
726 732
727 733 print 'Writing the file: %s'%self.filename
728 734
729 735 self.__writeFirstHeader()
730 736
731 737 return 1
732 738
733 739 def __setNewBlock(self):
740
734 741 if self.__fp == None:
735 return 0
742 self.__setNextFile()
736 743
737 744 if self.__flagIsNewFile:
738 745 return 1
739 746
740 747 if self.__blocksCounter < self.m_ProcessingHeader.dataBlocksPerFile:
741 748 self.__writeBasicHeader()
742 749 return 1
743 750
744 751 if not(self.__setNextFile()):
745 752 return 0
746 753
747 754 return 1
748 755
749 756 def __writeBlock(self):
750
751 self.__buffer.tofile(self.__fp)
752 757
753 self.__buffer = numpy.array([],self.__dataType)
758 data = numpy.zeros(self.__shapeBuffer, self.__dataType)
759
760 data['real'] = self.__buffer.real
761 data['imag'] = self.__buffer.imag
762
763 data = data.reshape((-1))
764
765 data.tofile(self.__fp)
766
767 self.__buffer.fill(0)
754 768
755 769 self.__buffer_id = 0
756 770
757 771 self.__flagIsNewFile = 0
758 772
759 773 self.flagIsNewBlock = 1
760 774
761 775 self.nWriteBlocks += 1
762 776
763 777 self.__blocksCounter += 1
764 778
765 779
766 780 def writeNextBlock(self):
767 781
768 782 if not(self.__setNewBlock()):
769 783 return 0
770 784
771 785 self.__writeBlock()
772 786
773 787 return 1
774 788
775 789 def __hasAllDataInBuffer(self):
776 790 if self.__buffer_id >= self.m_ProcessingHeader.profilesPerBlock:
777 791 return 1
778 792
779 793 return 0
780 794
781 795 def putData(self):
796
782 797 self.flagIsNewBlock = 0
783 798
784 799 if self.m_Voltage.flagNoData:
785 return None
800 return 0
786 801
787 802 if self.m_Voltage.flagResetProcessing:
788 self.__buffer = numpy.array([],self.__dataType)
803
804 self.__buffer.fill(0)
805
789 806 self.__buffer_id = 0
790 807 self.__setNextFile()
791 808
792 self.__setHeaderByObj()
793
794 shape = self.m_Voltage.data.shape
795 data = numpy.zeros(shape,self.__dataType)
796
797 data['real'] = self.m_Voltage.data.real
798 data['imag'] = self.m_Voltage.data.imag
799
800 data = data.reshape((-1))
801
802 self.__buffer = numpy.hstack((self.__buffer,data))
809 self.__buffer[self.__buffer_id,:,:] = self.m_Voltage.data
803 810
804 811 self.__buffer_id += 1
805 812
806 813 if self.__hasAllDataInBuffer():
814
815 self.__getHeader()
807 816 self.writeNextBlock()
808 817
809
810 818 if self.noMoreFiles:
811 print 'Process finished'
812 return None
819 #print 'Process finished'
820 return 0
813 821
814 822 return 1
815 823
816 def __setHeaderByObj(self):
824 def __getHeader(self):
817 825 self.m_BasicHeader = self.m_Voltage.m_BasicHeader.copy()
818 826 self.m_SystemHeader = self.m_Voltage.m_SystemHeader.copy()
819 827 self.m_RadarControllerHeader = self.m_Voltage.m_RadarControllerHeader.copy()
820 828 self.m_ProcessingHeader = self.m_Voltage.m_ProcessingHeader.copy()
821 829 self.__dataType = self.m_Voltage.dataType
822 830
823 831 def __setHeaderByFile(self):
824 832
825 833 format = self.__format
826 834 header = ['Basic','System','RadarController','Processing']
827 835
828 836 fmtFromFile = None
829 837 headerFromFile = None
830 838
831 839
832 840 fileTable = self.__configHeaderFile
833 841
834 842 if os.access(fileTable, os.R_OK):
835 843 import re, string
836 844
837 845 f = open(fileTable,'r')
838 846 lines = f.read()
839 847 f.close()
840 848
841 849 #Delete comments into expConfig
842 850 while 1:
843 851
844 852 startComment = string.find(lines.lower(),'#')
845 853 if startComment == -1:
846 854 break
847 855 endComment = string.find(lines.lower(),'\n',startComment)
848 856 lines = string.replace(lines,lines[startComment:endComment+1],'', 1)
849 857
850 858 while expFromFile == None:
851 859
852 860 currFmt = string.find(lines.lower(),'format="%s"' %(expName))
853 861 nextFmt = string.find(lines.lower(),'format',currFmt+10)
854 862
855 863 if currFmt == -1:
856 864 break
857 865 if nextFmt == -1:
858 866 nextFmt = len(lines)-1
859 867
860 868 fmtTable = lines[currFmt:nextFmt]
861 869 lines = lines[nextFmt:]
862 870
863 871 fmtRead = self.__getValueFromArg(fmtTable,'format')
864 872 if fmtRead != format:
865 873 continue
866 874 fmtFromFile = fmtRead
867 875
868 876 lines2 = fmtTable
869 877
870 878 while headerFromFile == None:
871 879
872 880 currHeader = string.find(lines2.lower(),'header="%s"' %(header))
873 881 nextHeader = string.find(lines2.lower(),'header',currHeader+10)
874 882
875 883 if currHeader == -1:
876 884 break
877 885 if nextHeader == -1:
878 886 nextHeader = len(lines2)-1
879 887
880 888 headerTable = lines2[currHeader:nextHeader]
881 889 lines2 = lines2[nextHeader:]
882 890
883 891 headerRead = self.__getValueFromArg(headerTable,'site')
884 892 if not(headerRead in header):
885 893 continue
886 894 headerFromFile = headerRead
887 895
888 896 if headerRead == 'Basic':
889 897 self.m_BasicHeader.size = self.__getValueFromArg(headerTable,'size',lower=False)
890 898 self.m_BasicHeader.version = self.__getValueFromArg(headerTable,'version',lower=False)
891 899 self.m_BasicHeader.dataBlock = self.__getValueFromArg(headerTable,'dataBlock',lower=False)
892 900 self.m_BasicHeader.utc = self.__getValueFromArg(headerTable,'utc',lower=False)
893 901 self.m_BasicHeader.miliSecond = self.__getValueFromArg(headerTable,'miliSecond',lower=False)
894 902 self.m_BasicHeader.timeZone = self.__getValueFromArg(headerTable,'timeZone',lower=False)
895 903 self.m_BasicHeader.dstFlag = self.__getValueFromArg(headerTable,'dstFlag',lower=False)
896 904 self.m_BasicHeader.errorCount = self.__getValueFromArg(headerTable,'errorCount',lower=False)
897 905
898 906 else:
899 907 print "file access denied:%s"%fileTable
900 908 sys.exit(0)
901 909
902 def setup(self,path,set=0,format='rawdata'):
910 def setup(self, path, set=0, format='rawdata'):
903 911
904 912
905 913 if format == 'hdf5':
906 914 ext = '.hdf5'
907 915 format = 'hdf5'
908 916 print 'call hdf5 library'
909 917 return 0
910 918
911 919 if format == 'rawdata':
912 920 ext = '.r'
913 921 format = 'Jicamarca'
914 922
915 923 #call to config_headers
916 924 #self.__setHeaderByFile()
917 self.__setHeaderByObj()
925
918 926 self.__path = path
919 927 self.__setFile = set - 1
920 928 self.__ext = ext
921 929 self.__format = format
922 self.__buffer = numpy.array([],self.__dataType)
923 930
924 if not(self.__setNextFile()):
925 return 0
931 self.__getHeader()
932 self.__shapeBuffer = (self.m_ProcessingHeader.profilesPerBlock,
933 self.m_ProcessingHeader.numHeights,
934 self.m_SystemHeader.numChannels )
935
936 self.__buffer = numpy.zeros(self.__shapeBuffer, numpy.dtype('complex'))
926 937
938 # if not(self.__setNextFile()):
939 # return 0
940 return 1
941
942
927 943
928 944
929 945 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,59 +1,74
1 1 '''
2 2 Created on 23/01/2012
3 3
4 4 @author $Author$
5 5 @version $Id$
6 6 '''
7 7 import os, sys
8 8 import time, datetime
9 9
10 10 from Model.Voltage import Voltage
11 from IO.VoltageIO import VoltageReader
11 from IO.VoltageIO import *
12 12 from Graphics.VoltagePlot import Osciloscope
13 13
14 14 class TestSChain():
15 15
16 16
17 17 def __init__(self):
18 18 self.setValues()
19 19 self.createObjects()
20 20 self.testSChain()
21 21 pass
22 22
23 23 def setValues(self):
24 24
25 25 self.path = '/home/roj-idl71/Data/RAWDATA/DP_Faraday/'
26 26 self.path = '/Users/danielangelsuarezmunoz/Documents/Projects/testWR'
27 self.path = '/home/roj-idl71/Data/RAWDATA/IMAGING'
28 self.path = '/home/roj-idl71/tmp/data'
27 29 #self.path = '/remote/puma/2004_11/DVD/'
30
31 self.ppath = "/home/roj-idl71/tmp/data"
28 32 self.startDateTime = datetime.datetime(2004,5,1,17,49,0)
29 33 self.endDateTime = datetime.datetime(2012,5,1,18,10,0)
30 34
31 35 def createObjects(self):
32 36
33 37 self.voltageObj = Voltage()
34 38 self.readerObj = VoltageReader(self.voltageObj)
35 39 self.plotObj = Osciloscope(self.voltageObj)
40 self.writerObj = VoltageWriter(self.voltageObj)
36 41
37 self.readerObj.setup(self.path, self.startDateTime, self.endDateTime)
42 if not(self.readerObj.setup(self.path, self.startDateTime, self.endDateTime)):
43 sys.exit(0)
44
45 if not(self.writerObj.setup(self.ppath)):
46 sys.exit(0)
38 47
39 48 def testSChain(self):
40 49
50 ini = time.time()
41 51 while(True):
42
43 52 self.readerObj.getData()
44 53 self.plotObj.plotData(idProfile = 1, type='iq', ymin = -100, ymax = 100)
45 54
46 if self.readerObj.flagResetProcessing:
47 print 'jump'
48
49 # if self.readerObj.flagIsNewBlock:
50 # print 'Block No %04d, Time: %s'%(self.readerObj.nReadBlocks,
51 # datetime.datetime.fromtimestamp(self.readerObj.m_BasicHeader.utc))
52
55 # self.writerObj.putData()
56
53 57 if self.readerObj.noMoreFiles:
54 58 break
55
59
60 if self.readerObj.flagIsNewBlock:
61 print 'Block No %04d, Time: %s' %(self.readerObj.nReadBlocks,
62 datetime.datetime.fromtimestamp(self.readerObj.m_BasicHeader.utc),)
63 fin = time.time()
64 print 'Tiempo de un bloque leido y escrito: [%6.5f]' %(fin - ini)
65 ini = time.time()
66
67
68
69
70
56 71 self.plotObj.end()
57 72
58 73 if __name__ == '__main__':
59 74 TestSChain() No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now