##// END OF EJS Templates
schain
RSS
Modificaciones del tipo de dato
Daniel Valdez -
r276:98940ac0422d
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,2579 +1,2580
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $
5 5 '''
6 6
7 7 import os, sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13
14 14 from jrodata import *
15 15 from jroheaderIO import *
16 16 from jroprocessing import *
17 17
18 18 LOCALTIME = -18000
19 19
20 20 def isNumber(str):
21 21 """
22 22 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
23 23
24 24 Excepciones:
25 25 Si un determinado string no puede ser convertido a numero
26 26 Input:
27 27 str, string al cual se le analiza para determinar si convertible a un numero o no
28 28
29 29 Return:
30 30 True : si el string es uno numerico
31 31 False : no es un string numerico
32 32 """
33 33 try:
34 34 float( str )
35 35 return True
36 36 except:
37 37 return False
38 38
39 39 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
40 40 """
41 41 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
42 42
43 43 Inputs:
44 44 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
45 45
46 46 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
47 47 segundos contados desde 01/01/1970.
48 48 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
49 49 segundos contados desde 01/01/1970.
50 50
51 51 Return:
52 52 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
53 53 fecha especificado, de lo contrario retorna False.
54 54
55 55 Excepciones:
56 56 Si el archivo no existe o no puede ser abierto
57 57 Si la cabecera no puede ser leida.
58 58
59 59 """
60 60 basicHeaderObj = BasicHeader(LOCALTIME)
61 61
62 62 try:
63 63 fp = open(filename,'rb')
64 64 except:
65 65 raise IOError, "The file %s can't be opened" %(filename)
66 66
67 67 sts = basicHeaderObj.read(fp)
68 68 fp.close()
69 69
70 70 if not(sts):
71 71 print "Skipping the file %s because it has not a valid header" %(filename)
72 72 return 0
73 73
74 74 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
75 75 return 0
76 76
77 77 return 1
78 78
79 79 def isFileinThisTime(filename, startTime, endTime):
80 80 """
81 81 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
82 82
83 83 Inputs:
84 84 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
85 85
86 86 startTime : tiempo inicial del rango seleccionado en formato datetime.time
87 87
88 88 endTime : tiempo final del rango seleccionado en formato datetime.time
89 89
90 90 Return:
91 91 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
92 92 fecha especificado, de lo contrario retorna False.
93 93
94 94 Excepciones:
95 95 Si el archivo no existe o no puede ser abierto
96 96 Si la cabecera no puede ser leida.
97 97
98 98 """
99 99
100 100
101 101 try:
102 102 fp = open(filename,'rb')
103 103 except:
104 104 raise IOError, "The file %s can't be opened" %(filename)
105 105
106 106 basicHeaderObj = BasicHeader(LOCALTIME)
107 107 sts = basicHeaderObj.read(fp)
108 108 fp.close()
109 109
110 110 thisTime = basicHeaderObj.datatime.time()
111 111
112 112 if not(sts):
113 113 print "Skipping the file %s because it has not a valid header" %(filename)
114 114 return 0
115 115
116 116 if not ((startTime <= thisTime) and (endTime > thisTime)):
117 117 return 0
118 118
119 119 return 1
120 120
121 121 def getlastFileFromPath(path, ext):
122 122 """
123 123 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
124 124 al final de la depuracion devuelve el ultimo file de la lista que quedo.
125 125
126 126 Input:
127 127 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
128 128 ext : extension de los files contenidos en una carpeta
129 129
130 130 Return:
131 131 El ultimo file de una determinada carpeta, no se considera el path.
132 132 """
133 133 validFilelist = []
134 134 fileList = os.listdir(path)
135 135
136 136 # 0 1234 567 89A BCDE
137 137 # H YYYY DDD SSS .ext
138 138
139 139 for file in fileList:
140 140 try:
141 141 year = int(file[1:5])
142 142 doy = int(file[5:8])
143 143
144 144
145 145 except:
146 146 continue
147 147
148 148 if (os.path.splitext(file)[-1].lower() != ext.lower()):
149 149 continue
150 150
151 151 validFilelist.append(file)
152 152
153 153 if validFilelist:
154 154 validFilelist = sorted( validFilelist, key=str.lower )
155 155 return validFilelist[-1]
156 156
157 157 return None
158 158
159 159 def checkForRealPath(path, year, doy, set, ext):
160 160 """
161 161 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
162 162 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
163 163 el path exacto de un determinado file.
164 164
165 165 Example :
166 166 nombre correcto del file es .../.../D2009307/P2009307367.ext
167 167
168 168 Entonces la funcion prueba con las siguientes combinaciones
169 169 .../.../y2009307367.ext
170 170 .../.../Y2009307367.ext
171 171 .../.../x2009307/y2009307367.ext
172 172 .../.../x2009307/Y2009307367.ext
173 173 .../.../X2009307/y2009307367.ext
174 174 .../.../X2009307/Y2009307367.ext
175 175 siendo para este caso, la ultima combinacion de letras, identica al file buscado
176 176
177 177 Return:
178 178 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
179 179 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
180 180 para el filename
181 181 """
182 182 fullfilename = None
183 183 find_flag = False
184 184 filename = None
185 185
186 186 prefixDirList = [None,'d','D']
187 187 if ext.lower() == ".r": #voltage
188 188 prefixFileList = ['d','D']
189 189 elif ext.lower() == ".pdata": #spectra
190 190 prefixFileList = ['p','P']
191 191 else:
192 192 return None, filename
193 193
194 194 #barrido por las combinaciones posibles
195 195 for prefixDir in prefixDirList:
196 196 thispath = path
197 197 if prefixDir != None:
198 198 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
199 199 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
200 200
201 201 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
202 202 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
203 203 fullfilename = os.path.join( thispath, filename ) #formo el path completo
204 204
205 205 if os.path.exists( fullfilename ): #verifico que exista
206 206 find_flag = True
207 207 break
208 208 if find_flag:
209 209 break
210 210
211 211 if not(find_flag):
212 212 return None, filename
213 213
214 214 return fullfilename, filename
215 215
216 216 class JRODataIO:
217 217
218 218 c = 3E8
219 219
220 220 isConfig = False
221 221
222 222 basicHeaderObj = BasicHeader(LOCALTIME)
223 223
224 224 systemHeaderObj = SystemHeader()
225 225
226 226 radarControllerHeaderObj = RadarControllerHeader()
227 227
228 228 processingHeaderObj = ProcessingHeader()
229 229
230 230 online = 0
231 231
232 232 dtype = None
233 233
234 234 pathList = []
235 235
236 236 filenameList = []
237 237
238 238 filename = None
239 239
240 240 ext = None
241 241
242 242 flagIsNewFile = 1
243 243
244 244 flagTimeBlock = 0
245 245
246 246 flagIsNewBlock = 0
247 247
248 248 fp = None
249 249
250 250 firstHeaderSize = 0
251 251
252 252 basicHeaderSize = 24
253 253
254 254 versionFile = 1103
255 255
256 256 fileSize = None
257 257
258 258 ippSeconds = None
259 259
260 260 fileSizeByHeader = None
261 261
262 262 fileIndex = None
263 263
264 264 profileIndex = None
265 265
266 266 blockIndex = None
267 267
268 268 nTotalBlocks = None
269 269
270 270 maxTimeStep = 30
271 271
272 272 lastUTTime = None
273 273
274 274 datablock = None
275 275
276 276 dataOut = None
277 277
278 278 blocksize = None
279 279
280 280 def __init__(self):
281 281
282 282 raise ValueError, "Not implemented"
283 283
284 284 def run(self):
285 285
286 286 raise ValueError, "Not implemented"
287 287
288 288 def getOutput(self):
289 289
290 290 return self.dataOut
291 291
292 292 class JRODataReader(JRODataIO, ProcessingUnit):
293 293
294 294 nReadBlocks = 0
295 295
296 296 delay = 10 #number of seconds waiting a new file
297 297
298 298 nTries = 3 #quantity tries
299 299
300 300 nFiles = 3 #number of files for searching
301 301
302 302 flagNoMoreFiles = 0
303 303
304 304 def __init__(self):
305 305
306 306 """
307 307
308 308 """
309 309
310 310 raise ValueError, "This method has not been implemented"
311 311
312 312
313 313 def createObjByDefault(self):
314 314 """
315 315
316 316 """
317 317 raise ValueError, "This method has not been implemented"
318 318
319 319 def getBlockDimension(self):
320 320
321 321 raise ValueError, "No implemented"
322 322
323 323 def __searchFilesOffLine(self,
324 324 path,
325 325 startDate,
326 326 endDate,
327 327 startTime=datetime.time(0,0,0),
328 328 endTime=datetime.time(23,59,59),
329 329 set=None,
330 330 expLabel='',
331 331 ext='.r',
332 332 walk=True):
333 333
334 334 pathList = []
335 335
336 336 if not walk:
337 337 pathList.append(path)
338 338
339 339 else:
340 340 dirList = []
341 341 for thisPath in os.listdir(path):
342 342 if os.path.isdir(os.path.join(path,thisPath)):
343 343 dirList.append(thisPath)
344 344
345 345 if not(dirList):
346 346 return None, None
347 347
348 348 thisDate = startDate
349 349
350 350 while(thisDate <= endDate):
351 351 year = thisDate.timetuple().tm_year
352 352 doy = thisDate.timetuple().tm_yday
353 353
354 354 match = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy))
355 355 if len(match) == 0:
356 356 thisDate += datetime.timedelta(1)
357 357 continue
358 358
359 359 pathList.append(os.path.join(path,match[0],expLabel))
360 360 thisDate += datetime.timedelta(1)
361 361
362 362 if pathList == []:
363 363 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
364 364 return None, None
365 365
366 366 print "%d folder(s) was(were) found for the date range: %s-%s" %(len(pathList), startDate, endDate)
367 367
368 368 filenameList = []
369 369 for thisPath in pathList:
370 370
371 371 fileList = glob.glob1(thisPath, "*%s" %ext)
372 372 fileList.sort()
373 373
374 374 for file in fileList:
375 375
376 376 filename = os.path.join(thisPath,file)
377 377
378 378 if isFileinThisTime(filename, startTime, endTime):
379 379 filenameList.append(filename)
380 380
381 381 if not(filenameList):
382 382 print "Any file was found for the time range %s - %s" %(startTime, endTime)
383 383 return None, None
384 384
385 385 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
386 386
387 387 self.filenameList = filenameList
388 388
389 389 return pathList, filenameList
390 390
391 391 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True):
392 392
393 393 """
394 394 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
395 395 devuelve el archivo encontrado ademas de otros datos.
396 396
397 397 Input:
398 398 path : carpeta donde estan contenidos los files que contiene data
399 399
400 400 expLabel : Nombre del subexperimento (subfolder)
401 401
402 402 ext : extension de los files
403 403
404 404 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
405 405
406 406 Return:
407 407 directory : eL directorio donde esta el file encontrado
408 408 filename : el ultimo file de una determinada carpeta
409 409 year : el anho
410 410 doy : el numero de dia del anho
411 411 set : el set del archivo
412 412
413 413
414 414 """
415 415 dirList = []
416 416
417 417 if walk:
418 418
419 419 #Filtra solo los directorios
420 420 for thisPath in os.listdir(path):
421 421 if os.path.isdir(os.path.join(path, thisPath)):
422 422 dirList.append(thisPath)
423 423
424 424 if not(dirList):
425 425 return None, None, None, None, None
426 426
427 427 dirList = sorted( dirList, key=str.lower )
428 428
429 429 doypath = dirList[-1]
430 430 fullpath = os.path.join(path, doypath, expLabel)
431 431
432 432 else:
433 433 fullpath = path
434 434
435 435 filename = getlastFileFromPath(fullpath, ext)
436 436
437 437 if not(filename):
438 438 return None, None, None, None, None
439 439
440 440 if not(self.__verifyFile(os.path.join(fullpath, filename))):
441 441 return None, None, None, None, None
442 442
443 443 year = int( filename[1:5] )
444 444 doy = int( filename[5:8] )
445 445 set = int( filename[8:11] )
446 446
447 447 return fullpath, filename, year, doy, set
448 448
449 449
450 450
451 451 def __setNextFileOffline(self):
452 452
453 453 idFile = self.fileIndex
454 454
455 455 while (True):
456 456 idFile += 1
457 457 if not(idFile < len(self.filenameList)):
458 458 self.flagNoMoreFiles = 1
459 459 print "No more Files"
460 460 return 0
461 461
462 462 filename = self.filenameList[idFile]
463 463
464 464 if not(self.__verifyFile(filename)):
465 465 continue
466 466
467 467 fileSize = os.path.getsize(filename)
468 468 fp = open(filename,'rb')
469 469 break
470 470
471 471 self.flagIsNewFile = 1
472 472 self.fileIndex = idFile
473 473 self.filename = filename
474 474 self.fileSize = fileSize
475 475 self.fp = fp
476 476
477 477 print "Setting the file: %s"%self.filename
478 478
479 479 return 1
480 480
481 481 def __setNextFileOnline(self):
482 482 """
483 483 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
484 484 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
485 485 siguientes.
486 486
487 487 Affected:
488 488 self.flagIsNewFile
489 489 self.filename
490 490 self.fileSize
491 491 self.fp
492 492 self.set
493 493 self.flagNoMoreFiles
494 494
495 495 Return:
496 496 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
497 497 1 : si el file fue abierto con exito y esta listo a ser leido
498 498
499 499 Excepciones:
500 500 Si un determinado file no puede ser abierto
501 501 """
502 502 nFiles = 0
503 503 fileOk_flag = False
504 504 firstTime_flag = True
505 505
506 506 self.set += 1
507 507
508 508 #busca el 1er file disponible
509 509 fullfilename, filename = checkForRealPath( self.path, self.year, self.doy, self.set, self.ext )
510 510 if fullfilename:
511 511 if self.__verifyFile(fullfilename, False):
512 512 fileOk_flag = True
513 513
514 514 #si no encuentra un file entonces espera y vuelve a buscar
515 515 if not(fileOk_flag):
516 516 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
517 517
518 518 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
519 519 tries = self.nTries
520 520 else:
521 521 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
522 522
523 523 for nTries in range( tries ):
524 524 if firstTime_flag:
525 525 print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
526 526 time.sleep( self.delay )
527 527 else:
528 528 print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
529 529
530 530 fullfilename, filename = checkForRealPath( self.path, self.year, self.doy, self.set, self.ext )
531 531 if fullfilename:
532 532 if self.__verifyFile(fullfilename):
533 533 fileOk_flag = True
534 534 break
535 535
536 536 if fileOk_flag:
537 537 break
538 538
539 539 firstTime_flag = False
540 540
541 541 print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename
542 542 self.set += 1
543 543
544 544 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
545 545 self.set = 0
546 546 self.doy += 1
547 547
548 548 if fileOk_flag:
549 549 self.fileSize = os.path.getsize( fullfilename )
550 550 self.filename = fullfilename
551 551 self.flagIsNewFile = 1
552 552 if self.fp != None: self.fp.close()
553 553 self.fp = open(fullfilename, 'rb')
554 554 self.flagNoMoreFiles = 0
555 555 print 'Setting the file: %s' % fullfilename
556 556 else:
557 557 self.fileSize = 0
558 558 self.filename = None
559 559 self.flagIsNewFile = 0
560 560 self.fp = None
561 561 self.flagNoMoreFiles = 1
562 562 print 'No more Files'
563 563
564 564 return fileOk_flag
565 565
566 566
567 567 def setNextFile(self):
568 568 if self.fp != None:
569 569 self.fp.close()
570 570
571 571 if self.online:
572 572 newFile = self.__setNextFileOnline()
573 573 else:
574 574 newFile = self.__setNextFileOffline()
575 575
576 576 if not(newFile):
577 577 return 0
578 578
579 579 self.__readFirstHeader()
580 580 self.nReadBlocks = 0
581 581 return 1
582 582
583 583 def __waitNewBlock(self):
584 584 """
585 585 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
586 586
587 587 Si el modo de lectura es OffLine siempre retorn 0
588 588 """
589 589 if not self.online:
590 590 return 0
591 591
592 592 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
593 593 return 0
594 594
595 595 currentPointer = self.fp.tell()
596 596
597 597 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
598 598
599 599 for nTries in range( self.nTries ):
600 600
601 601 self.fp.close()
602 602 self.fp = open( self.filename, 'rb' )
603 603 self.fp.seek( currentPointer )
604 604
605 605 self.fileSize = os.path.getsize( self.filename )
606 606 currentSize = self.fileSize - currentPointer
607 607
608 608 if ( currentSize >= neededSize ):
609 609 self.__rdBasicHeader()
610 610 return 1
611 611
612 612 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
613 613 time.sleep( self.delay )
614 614
615 615
616 616 return 0
617 617
618 618 def __setNewBlock(self):
619 619
620 620 if self.fp == None:
621 621 return 0
622 622
623 623 if self.flagIsNewFile:
624 624 return 1
625 625
626 626 self.lastUTTime = self.basicHeaderObj.utc
627 627 currentSize = self.fileSize - self.fp.tell()
628 628 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
629 629
630 630 if (currentSize >= neededSize):
631 631 self.__rdBasicHeader()
632 632 return 1
633 633
634 634 if self.__waitNewBlock():
635 635 return 1
636 636
637 637 if not(self.setNextFile()):
638 638 return 0
639 639
640 640 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
641 641
642 642 self.flagTimeBlock = 0
643 643
644 644 if deltaTime > self.maxTimeStep:
645 645 self.flagTimeBlock = 1
646 646
647 647 return 1
648 648
649 649
650 650 def readNextBlock(self):
651 651 if not(self.__setNewBlock()):
652 652 return 0
653 653
654 654 if not(self.readBlock()):
655 655 return 0
656 656
657 657 return 1
658 658
659 659 def __rdProcessingHeader(self, fp=None):
660 660 if fp == None:
661 661 fp = self.fp
662 662
663 663 self.processingHeaderObj.read(fp)
664 664
665 665 def __rdRadarControllerHeader(self, fp=None):
666 666 if fp == None:
667 667 fp = self.fp
668 668
669 669 self.radarControllerHeaderObj.read(fp)
670 670
671 671 def __rdSystemHeader(self, fp=None):
672 672 if fp == None:
673 673 fp = self.fp
674 674
675 675 self.systemHeaderObj.read(fp)
676 676
677 677 def __rdBasicHeader(self, fp=None):
678 678 if fp == None:
679 679 fp = self.fp
680 680
681 681 self.basicHeaderObj.read(fp)
682 682
683 683
684 684 def __readFirstHeader(self):
685 685 self.__rdBasicHeader()
686 686 self.__rdSystemHeader()
687 687 self.__rdRadarControllerHeader()
688 688 self.__rdProcessingHeader()
689 689
690 690 self.firstHeaderSize = self.basicHeaderObj.size
691 691
692 692 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
693 693 if datatype == 0:
694 694 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
695 695 elif datatype == 1:
696 696 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
697 697 elif datatype == 2:
698 698 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
699 699 elif datatype == 3:
700 700 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
701 701 elif datatype == 4:
702 702 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
703 703 elif datatype == 5:
704 704 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
705 705 else:
706 706 raise ValueError, 'Data type was not defined'
707 707
708 708 self.dtype = datatype_str
709 709 self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
710 710 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
711 711 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
712 712 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
713 713 self.getBlockDimension()
714 714
715 715
716 716 def __verifyFile(self, filename, msgFlag=True):
717 717 msg = None
718 718 try:
719 719 fp = open(filename, 'rb')
720 720 currentPosition = fp.tell()
721 721 except:
722 722 if msgFlag:
723 723 print "The file %s can't be opened" % (filename)
724 724 return False
725 725
726 726 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
727 727
728 728 if neededSize == 0:
729 729 basicHeaderObj = BasicHeader(LOCALTIME)
730 730 systemHeaderObj = SystemHeader()
731 731 radarControllerHeaderObj = RadarControllerHeader()
732 732 processingHeaderObj = ProcessingHeader()
733 733
734 734 try:
735 735 if not( basicHeaderObj.read(fp) ): raise IOError
736 736 if not( systemHeaderObj.read(fp) ): raise IOError
737 737 if not( radarControllerHeaderObj.read(fp) ): raise IOError
738 738 if not( processingHeaderObj.read(fp) ): raise IOError
739 739 data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
740 740
741 741 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
742 742
743 743 except:
744 744 if msgFlag:
745 745 print "\tThe file %s is empty or it hasn't enough data" % filename
746 746
747 747 fp.close()
748 748 return False
749 749 else:
750 750 msg = "\tSkipping the file %s due to it hasn't enough data" %filename
751 751
752 752 fp.close()
753 753 fileSize = os.path.getsize(filename)
754 754 currentSize = fileSize - currentPosition
755 755 if currentSize < neededSize:
756 756 if msgFlag and (msg != None):
757 757 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
758 758 return False
759 759
760 760 return True
761 761
762 762 def setup(self,
763 763 path=None,
764 764 startDate=None,
765 765 endDate=None,
766 766 startTime=datetime.time(0,0,0),
767 767 endTime=datetime.time(23,59,59),
768 768 set=0,
769 769 expLabel = "",
770 770 ext = None,
771 771 online = False,
772 772 delay = 60,
773 773 walk = True):
774 774
775 775 if path == None:
776 776 raise ValueError, "The path is not valid"
777 777
778 778 if ext == None:
779 779 ext = self.ext
780 780
781 781 if online:
782 782 print "Searching files in online mode..."
783 783
784 784 for nTries in range( self.nTries ):
785 785 fullpath, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk)
786 786
787 787 if fullpath:
788 788 break
789 789
790 790 print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
791 791 time.sleep( self.delay )
792 792
793 793 if not(fullpath):
794 794 print "There 'isn't valied files in %s" % path
795 795 return None
796 796
797 797 self.year = year
798 798 self.doy = doy
799 799 self.set = set - 1
800 800 self.path = path
801 801
802 802 else:
803 803 print "Searching files in offline mode ..."
804 804 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
805 805 startTime=startTime, endTime=endTime,
806 806 set=set, expLabel=expLabel, ext=ext,
807 807 walk=walk)
808 808
809 809 if not(pathList):
810 810 print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
811 811 datetime.datetime.combine(startDate,startTime).ctime(),
812 812 datetime.datetime.combine(endDate,endTime).ctime())
813 813
814 814 sys.exit(-1)
815 815
816 816
817 817 self.fileIndex = -1
818 818 self.pathList = pathList
819 819 self.filenameList = filenameList
820 820
821 821 self.online = online
822 822 self.delay = delay
823 823 ext = ext.lower()
824 824 self.ext = ext
825 825
826 826 if not(self.setNextFile()):
827 827 if (startDate!=None) and (endDate!=None):
828 828 print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
829 829 elif startDate != None:
830 830 print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
831 831 else:
832 832 print "No files"
833 833
834 834 sys.exit(-1)
835 835
836 836 # self.updateDataHeader()
837 837
838 838 return self.dataOut
839 839
840 840 def getData():
841 841
842 842 raise ValueError, "This method has not been implemented"
843 843
844 844 def hasNotDataInBuffer():
845 845
846 846 raise ValueError, "This method has not been implemented"
847 847
848 848 def readBlock():
849 849
850 850 raise ValueError, "This method has not been implemented"
851 851
852 852 def isEndProcess(self):
853 853
854 854 return self.flagNoMoreFiles
855 855
856 856 def printReadBlocks(self):
857 857
858 858 print "Number of read blocks per file %04d" %self.nReadBlocks
859 859
860 860 def printTotalBlocks(self):
861 861
862 862 print "Number of read blocks %04d" %self.nTotalBlocks
863 863
864 864 def printNumberOfBlock(self):
865 865
866 866 if self.flagIsNewBlock:
867 867 print "Block No. %04d, Total blocks %04d" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks)
868 868
869 869 def printInfo(self):
870 870
871 871 print self.basicHeaderObj.printInfo()
872 872 print self.systemHeaderObj.printInfo()
873 873 print self.radarControllerHeaderObj.printInfo()
874 874 print self.processingHeaderObj.printInfo()
875 875
876 876
877 877 def run(self, **kwargs):
878 878
879 879 if not(self.isConfig):
880 880
881 881 # self.dataOut = dataOut
882 882 self.setup(**kwargs)
883 883 self.isConfig = True
884 884
885 885 self.getData()
886 886
887 887 class JRODataWriter(JRODataIO, Operation):
888 888
889 889 """
890 890 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
891 891 de los datos siempre se realiza por bloques.
892 892 """
893 893
894 894 blockIndex = 0
895 895
896 896 path = None
897 897
898 898 setFile = None
899 899
900 900 profilesPerBlock = None
901 901
902 902 blocksPerFile = None
903 903
904 904 nWriteBlocks = 0
905 905
906 906 def __init__(self, dataOut=None):
907 907 raise ValueError, "Not implemented"
908 908
909 909
910 910 def hasAllDataInBuffer(self):
911 911 raise ValueError, "Not implemented"
912 912
913 913
914 914 def setBlockDimension(self):
915 915 raise ValueError, "Not implemented"
916 916
917 917
918 918 def writeBlock(self):
919 919 raise ValueError, "No implemented"
920 920
921 921
922 922 def putData(self):
923 923 raise ValueError, "No implemented"
924 924
925 925 def getDataHeader(self):
926 926 """
927 927 Obtiene una copia del First Header
928 928
929 929 Affected:
930 930
931 931 self.basicHeaderObj
932 932 self.systemHeaderObj
933 933 self.radarControllerHeaderObj
934 934 self.processingHeaderObj self.
935 935
936 936 Return:
937 937 None
938 938 """
939 939
940 940 raise ValueError, "No implemented"
941 941
942 942 def getBasicHeader(self):
943 943
944 944 self.basicHeaderObj.size = self.basicHeaderSize #bytes
945 945 self.basicHeaderObj.version = self.versionFile
946 946 self.basicHeaderObj.dataBlock = self.nTotalBlocks
947 947
948 948 utc = numpy.floor(self.dataOut.utctime)
949 949 milisecond = (self.dataOut.utctime - utc)* 1000.0
950 950
951 951 self.basicHeaderObj.utc = utc
952 952 self.basicHeaderObj.miliSecond = milisecond
953 953 self.basicHeaderObj.timeZone = 0
954 954 self.basicHeaderObj.dstFlag = 0
955 955 self.basicHeaderObj.errorCount = 0
956 956
957 957 def __writeFirstHeader(self):
958 958 """
959 959 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
960 960
961 961 Affected:
962 962 __dataType
963 963
964 964 Return:
965 965 None
966 966 """
967 967
968 968 # CALCULAR PARAMETROS
969 969
970 970 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
971 971 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
972 972
973 973 self.basicHeaderObj.write(self.fp)
974 974 self.systemHeaderObj.write(self.fp)
975 975 self.radarControllerHeaderObj.write(self.fp)
976 976 self.processingHeaderObj.write(self.fp)
977 977
978 978 self.dtype = self.dataOut.dtype
979 979
980 980 def __setNewBlock(self):
981 981 """
982 982 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
983 983
984 984 Return:
985 985 0 : si no pudo escribir nada
986 986 1 : Si escribio el Basic el First Header
987 987 """
988 988 if self.fp == None:
989 989 self.setNextFile()
990 990
991 991 if self.flagIsNewFile:
992 992 return 1
993 993
994 994 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
995 995 self.basicHeaderObj.write(self.fp)
996 996 return 1
997 997
998 998 if not( self.setNextFile() ):
999 999 return 0
1000 1000
1001 1001 return 1
1002 1002
1003 1003
1004 1004 def writeNextBlock(self):
1005 1005 """
1006 1006 Selecciona el bloque siguiente de datos y los escribe en un file
1007 1007
1008 1008 Return:
1009 1009 0 : Si no hizo pudo escribir el bloque de datos
1010 1010 1 : Si no pudo escribir el bloque de datos
1011 1011 """
1012 1012 if not( self.__setNewBlock() ):
1013 1013 return 0
1014 1014
1015 1015 self.writeBlock()
1016 1016
1017 1017 return 1
1018 1018
1019 1019 def setNextFile(self):
1020 1020 """
1021 1021 Determina el siguiente file que sera escrito
1022 1022
1023 1023 Affected:
1024 1024 self.filename
1025 1025 self.subfolder
1026 1026 self.fp
1027 1027 self.setFile
1028 1028 self.flagIsNewFile
1029 1029
1030 1030 Return:
1031 1031 0 : Si el archivo no puede ser escrito
1032 1032 1 : Si el archivo esta listo para ser escrito
1033 1033 """
1034 1034 ext = self.ext
1035 1035 path = self.path
1036 1036
1037 1037 if self.fp != None:
1038 1038 self.fp.close()
1039 1039
1040 1040 timeTuple = time.localtime( self.dataOut.utctime)
1041 1041 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1042 1042
1043 1043 fullpath = os.path.join( path, subfolder )
1044 1044 if not( os.path.exists(fullpath) ):
1045 1045 os.mkdir(fullpath)
1046 1046 self.setFile = -1 #inicializo mi contador de seteo
1047 1047 else:
1048 1048 filesList = os.listdir( fullpath )
1049 1049 if len( filesList ) > 0:
1050 1050 filesList = sorted( filesList, key=str.lower )
1051 1051 filen = filesList[-1]
1052 1052 # el filename debera tener el siguiente formato
1053 1053 # 0 1234 567 89A BCDE (hex)
1054 1054 # x YYYY DDD SSS .ext
1055 1055 if isNumber( filen[8:11] ):
1056 1056 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1057 1057 else:
1058 1058 self.setFile = -1
1059 1059 else:
1060 1060 self.setFile = -1 #inicializo mi contador de seteo
1061 1061
1062 1062 setFile = self.setFile
1063 1063 setFile += 1
1064 1064
1065 1065 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
1066 1066 timeTuple.tm_year,
1067 1067 timeTuple.tm_yday,
1068 1068 setFile,
1069 1069 ext )
1070 1070
1071 1071 filename = os.path.join( path, subfolder, file )
1072 1072
1073 1073 fp = open( filename,'wb' )
1074 1074
1075 1075 self.blockIndex = 0
1076 1076
1077 1077 #guardando atributos
1078 1078 self.filename = filename
1079 1079 self.subfolder = subfolder
1080 1080 self.fp = fp
1081 1081 self.setFile = setFile
1082 1082 self.flagIsNewFile = 1
1083 1083
1084 1084 self.getDataHeader()
1085 1085
1086 1086 print 'Writing the file: %s'%self.filename
1087 1087
1088 1088 self.__writeFirstHeader()
1089 1089
1090 1090 return 1
1091 1091
1092 1092 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=None, set=0, ext=None):
1093 1093 """
1094 1094 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1095 1095
1096 1096 Inputs:
1097 1097 path : el path destino en el cual se escribiran los files a crear
1098 1098 format : formato en el cual sera salvado un file
1099 1099 set : el setebo del file
1100 1100
1101 1101 Return:
1102 1102 0 : Si no realizo un buen seteo
1103 1103 1 : Si realizo un buen seteo
1104 1104 """
1105 1105
1106 1106 if ext == None:
1107 1107 ext = self.ext
1108 1108
1109 1109 ext = ext.lower()
1110 1110
1111 1111 self.ext = ext
1112 1112
1113 1113 self.path = path
1114 1114
1115 1115 self.setFile = set - 1
1116 1116
1117 1117 self.blocksPerFile = blocksPerFile
1118 1118
1119 1119 self.profilesPerBlock = profilesPerBlock
1120 1120
1121 1121 self.dataOut = dataOut
1122 1122
1123 1123 if not(self.setNextFile()):
1124 1124 print "There isn't a next file"
1125 1125 return 0
1126 1126
1127 1127 self.setBlockDimension()
1128 1128
1129 1129 return 1
1130 1130
1131 1131 def run(self, dataOut, **kwargs):
1132 1132
1133 1133 if not(self.isConfig):
1134 1134
1135 1135 self.setup(dataOut, **kwargs)
1136 1136 self.isConfig = True
1137 1137
1138 1138 self.putData()
1139 1139
1140 1140 class VoltageReader(JRODataReader):
1141 1141 """
1142 1142 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
1143 1143 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
1144 1144 perfiles*alturas*canales) son almacenados en la variable "buffer".
1145 1145
1146 1146 perfiles * alturas * canales
1147 1147
1148 1148 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
1149 1149 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
1150 1150 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
1151 1151 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
1152 1152
1153 1153 Example:
1154 1154
1155 1155 dpath = "/home/myuser/data"
1156 1156
1157 1157 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
1158 1158
1159 1159 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
1160 1160
1161 1161 readerObj = VoltageReader()
1162 1162
1163 1163 readerObj.setup(dpath, startTime, endTime)
1164 1164
1165 1165 while(True):
1166 1166
1167 1167 #to get one profile
1168 1168 profile = readerObj.getData()
1169 1169
1170 1170 #print the profile
1171 1171 print profile
1172 1172
1173 1173 #If you want to see all datablock
1174 1174 print readerObj.datablock
1175 1175
1176 1176 if readerObj.flagNoMoreFiles:
1177 1177 break
1178 1178
1179 1179 """
1180 1180
1181 1181 ext = ".r"
1182 1182
1183 1183 optchar = "D"
1184 1184 dataOut = None
1185 1185
1186 1186
1187 1187 def __init__(self):
1188 1188 """
1189 1189 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
1190 1190
1191 1191 Input:
1192 1192 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
1193 1193 almacenar un perfil de datos cada vez que se haga un requerimiento
1194 1194 (getData). El perfil sera obtenido a partir del buffer de datos,
1195 1195 si el buffer esta vacio se hara un nuevo proceso de lectura de un
1196 1196 bloque de datos.
1197 1197 Si este parametro no es pasado se creara uno internamente.
1198 1198
1199 1199 Variables afectadas:
1200 1200 self.dataOut
1201 1201
1202 1202 Return:
1203 1203 None
1204 1204 """
1205 1205
1206 1206 self.isConfig = False
1207 1207
1208 1208 self.datablock = None
1209 1209
1210 1210 self.utc = 0
1211 1211
1212 1212 self.ext = ".r"
1213 1213
1214 1214 self.optchar = "D"
1215 1215
1216 1216 self.basicHeaderObj = BasicHeader(LOCALTIME)
1217 1217
1218 1218 self.systemHeaderObj = SystemHeader()
1219 1219
1220 1220 self.radarControllerHeaderObj = RadarControllerHeader()
1221 1221
1222 1222 self.processingHeaderObj = ProcessingHeader()
1223 1223
1224 1224 self.online = 0
1225 1225
1226 1226 self.fp = None
1227 1227
1228 1228 self.idFile = None
1229 1229
1230 1230 self.dtype = None
1231 1231
1232 1232 self.fileSizeByHeader = None
1233 1233
1234 1234 self.filenameList = []
1235 1235
1236 1236 self.filename = None
1237 1237
1238 1238 self.fileSize = None
1239 1239
1240 1240 self.firstHeaderSize = 0
1241 1241
1242 1242 self.basicHeaderSize = 24
1243 1243
1244 1244 self.pathList = []
1245 1245
1246 1246 self.filenameList = []
1247 1247
1248 1248 self.lastUTTime = 0
1249 1249
1250 1250 self.maxTimeStep = 30
1251 1251
1252 1252 self.flagNoMoreFiles = 0
1253 1253
1254 1254 self.set = 0
1255 1255
1256 1256 self.path = None
1257 1257
1258 1258 self.profileIndex = 9999
1259 1259
1260 1260 self.delay = 3 #seconds
1261 1261
1262 1262 self.nTries = 3 #quantity tries
1263 1263
1264 1264 self.nFiles = 3 #number of files for searching
1265 1265
1266 1266 self.nReadBlocks = 0
1267 1267
1268 1268 self.flagIsNewFile = 1
1269 1269
1270 1270 self.ippSeconds = 0
1271 1271
1272 1272 self.flagTimeBlock = 0
1273 1273
1274 1274 self.flagIsNewBlock = 0
1275 1275
1276 1276 self.nTotalBlocks = 0
1277 1277
1278 1278 self.blocksize = 0
1279 1279
1280 1280 self.dataOut = self.createObjByDefault()
1281 1281
1282 1282 def createObjByDefault(self):
1283 1283
1284 1284 dataObj = Voltage()
1285 1285
1286 1286 return dataObj
1287 1287
1288 1288 def __hasNotDataInBuffer(self):
1289 1289 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
1290 1290 return 1
1291 1291 return 0
1292 1292
1293 1293
1294 1294 def getBlockDimension(self):
1295 1295 """
1296 1296 Obtiene la cantidad de puntos a leer por cada bloque de datos
1297 1297
1298 1298 Affected:
1299 1299 self.blocksize
1300 1300
1301 1301 Return:
1302 1302 None
1303 1303 """
1304 1304 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
1305 1305 self.blocksize = pts2read
1306 1306
1307 1307
1308 1308 def readBlock(self):
1309 1309 """
1310 1310 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
1311 1311 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
1312 1312 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
1313 1313 es seteado a 0
1314 1314
1315 1315 Inputs:
1316 1316 None
1317 1317
1318 1318 Return:
1319 1319 None
1320 1320
1321 1321 Affected:
1322 1322 self.profileIndex
1323 1323 self.datablock
1324 1324 self.flagIsNewFile
1325 1325 self.flagIsNewBlock
1326 1326 self.nTotalBlocks
1327 1327
1328 1328 Exceptions:
1329 1329 Si un bloque leido no es un bloque valido
1330 1330 """
1331 1331
1332 1332 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
1333 1333
1334 1334 try:
1335 1335 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
1336 1336 except:
1337 1337 print "The read block (%3d) has not enough data" %self.nReadBlocks
1338 1338 return 0
1339 1339
1340 1340 junk = numpy.transpose(junk, (2,0,1))
1341 1341 self.datablock = junk['real'] + junk['imag']*1j
1342 1342
1343 1343 self.profileIndex = 0
1344 1344
1345 1345 self.flagIsNewFile = 0
1346 1346 self.flagIsNewBlock = 1
1347 1347
1348 1348 self.nTotalBlocks += 1
1349 1349 self.nReadBlocks += 1
1350 1350
1351 1351 return 1
1352 1352
1353 1353
1354 1354 def getData(self):
1355 1355 """
1356 1356 getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"
1357 1357 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
1358 1358 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
1359 1359
1360 1360 Ademas incrementa el contador del buffer en 1.
1361 1361
1362 1362 Return:
1363 1363 data : retorna un perfil de voltages (alturas * canales) copiados desde el
1364 1364 buffer. Si no hay mas archivos a leer retorna None.
1365 1365
1366 1366 Variables afectadas:
1367 1367 self.dataOut
1368 1368 self.profileIndex
1369 1369
1370 1370 Affected:
1371 1371 self.dataOut
1372 1372 self.profileIndex
1373 1373 self.flagTimeBlock
1374 1374 self.flagIsNewBlock
1375 1375 """
1376 1376
1377 1377 if self.flagNoMoreFiles:
1378 1378 self.dataOut.flagNoData = True
1379 1379 print 'Process finished'
1380 1380 return 0
1381 1381
1382 1382 self.flagTimeBlock = 0
1383 1383 self.flagIsNewBlock = 0
1384 1384
1385 1385 if self.__hasNotDataInBuffer():
1386 1386
1387 1387 if not( self.readNextBlock() ):
1388 1388 return 0
1389 1389
1390 self.dataOut.dtype = numpy.dtype([('real','<f8'),('imag','<f8')]) #self.dtype
1390 self.dataOut.dtype = self.dtype
1391 1391
1392 1392 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
1393 1393
1394 1394 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
1395 1395
1396 1396 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
1397 1397
1398 1398 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
1399 1399
1400 1400 self.dataOut.flagTimeBlock = self.flagTimeBlock
1401 1401
1402 1402 self.dataOut.ippSeconds = self.ippSeconds
1403 1403
1404 1404 self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt
1405 1405
1406 1406 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
1407 1407
1408 1408 self.dataOut.flagShiftFFT = False
1409 1409
1410 1410 if self.radarControllerHeaderObj.code != None:
1411 1411
1412 1412 self.dataOut.nCode = self.radarControllerHeaderObj.nCode
1413 1413
1414 1414 self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
1415 1415
1416 1416 self.dataOut.code = self.radarControllerHeaderObj.code
1417 1417
1418 1418 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
1419 1419
1420 1420 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
1421 1421
1422 1422 self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
1423 1423
1424 1424 self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip
1425 1425
1426 1426 self.dataOut.flagShiftFFT = False
1427 1427
1428 1428
1429 1429 # self.updateDataHeader()
1430 1430
1431 1431 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
1432 1432
1433 1433 if self.datablock == None:
1434 1434 self.dataOut.flagNoData = True
1435 1435 return 0
1436 1436
1437 1437 self.dataOut.data = self.datablock[:,self.profileIndex,:]
1438 1438
1439 1439 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds
1440 1440
1441 1441 self.profileIndex += 1
1442 1442
1443 1443 self.dataOut.flagNoData = False
1444 1444
1445 1445 # print self.profileIndex, self.dataOut.utctime
1446 1446 # if self.profileIndex == 800:
1447 1447 # a=1
1448 1448
1449 1449
1450 1450 return self.dataOut.data
1451 1451
1452 1452
1453 1453 class VoltageWriter(JRODataWriter):
1454 1454 """
1455 1455 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
1456 1456 de los datos siempre se realiza por bloques.
1457 1457 """
1458 1458
1459 1459 ext = ".r"
1460 1460
1461 1461 optchar = "D"
1462 1462
1463 1463 shapeBuffer = None
1464 1464
1465 1465
1466 1466 def __init__(self):
1467 1467 """
1468 1468 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
1469 1469
1470 1470 Affected:
1471 1471 self.dataOut
1472 1472
1473 1473 Return: None
1474 1474 """
1475 1475
1476 1476 self.nTotalBlocks = 0
1477 1477
1478 1478 self.profileIndex = 0
1479 1479
1480 1480 self.isConfig = False
1481 1481
1482 1482 self.fp = None
1483 1483
1484 1484 self.flagIsNewFile = 1
1485 1485
1486 1486 self.nTotalBlocks = 0
1487 1487
1488 1488 self.flagIsNewBlock = 0
1489 1489
1490 1490 self.setFile = None
1491 1491
1492 1492 self.dtype = None
1493 1493
1494 1494 self.path = None
1495 1495
1496 1496 self.filename = None
1497 1497
1498 1498 self.basicHeaderObj = BasicHeader(LOCALTIME)
1499 1499
1500 1500 self.systemHeaderObj = SystemHeader()
1501 1501
1502 1502 self.radarControllerHeaderObj = RadarControllerHeader()
1503 1503
1504 1504 self.processingHeaderObj = ProcessingHeader()
1505 1505
1506 1506 def hasAllDataInBuffer(self):
1507 1507 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
1508 1508 return 1
1509 1509 return 0
1510 1510
1511 1511
1512 1512 def setBlockDimension(self):
1513 1513 """
1514 1514 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
1515 1515
1516 1516 Affected:
1517 1517 self.shape_spc_Buffer
1518 1518 self.shape_cspc_Buffer
1519 1519 self.shape_dc_Buffer
1520 1520
1521 1521 Return: None
1522 1522 """
1523 1523 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
1524 1524 self.processingHeaderObj.nHeights,
1525 1525 self.systemHeaderObj.nChannels)
1526 1526
1527 1527 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
1528 1528 self.processingHeaderObj.profilesPerBlock,
1529 1529 self.processingHeaderObj.nHeights),
1530 dtype=numpy.dtype('complex'))
1530 dtype=numpy.dtype('complex64'))
1531 1531
1532 1532
1533 1533 def writeBlock(self):
1534 1534 """
1535 1535 Escribe el buffer en el file designado
1536 1536
1537 1537 Affected:
1538 1538 self.profileIndex
1539 1539 self.flagIsNewFile
1540 1540 self.flagIsNewBlock
1541 1541 self.nTotalBlocks
1542 1542 self.blockIndex
1543 1543
1544 1544 Return: None
1545 1545 """
1546 1546 data = numpy.zeros( self.shapeBuffer, self.dtype )
1547 1547
1548 1548 junk = numpy.transpose(self.datablock, (1,2,0))
1549 1549
1550 1550 data['real'] = junk.real
1551 1551 data['imag'] = junk.imag
1552 1552
1553 1553 data = data.reshape( (-1) )
1554 1554
1555 1555 data.tofile( self.fp )
1556 1556
1557 1557 self.datablock.fill(0)
1558 1558
1559 1559 self.profileIndex = 0
1560 1560 self.flagIsNewFile = 0
1561 1561 self.flagIsNewBlock = 1
1562 1562
1563 1563 self.blockIndex += 1
1564 1564 self.nTotalBlocks += 1
1565 1565
1566 1566 def putData(self):
1567 1567 """
1568 1568 Setea un bloque de datos y luego los escribe en un file
1569 1569
1570 1570 Affected:
1571 1571 self.flagIsNewBlock
1572 1572 self.profileIndex
1573 1573
1574 1574 Return:
1575 1575 0 : Si no hay data o no hay mas files que puedan escribirse
1576 1576 1 : Si se escribio la data de un bloque en un file
1577 1577 """
1578 1578 if self.dataOut.flagNoData:
1579 1579 return 0
1580 1580
1581 1581 self.flagIsNewBlock = 0
1582 1582
1583 1583 if self.dataOut.flagTimeBlock:
1584 1584
1585 1585 self.datablock.fill(0)
1586 1586 self.profileIndex = 0
1587 1587 self.setNextFile()
1588 1588
1589 1589 if self.profileIndex == 0:
1590 1590 self.getBasicHeader()
1591 1591
1592 1592 self.datablock[:,self.profileIndex,:] = self.dataOut.data
1593 1593
1594 1594 self.profileIndex += 1
1595 1595
1596 1596 if self.hasAllDataInBuffer():
1597 1597 #if self.flagIsNewFile:
1598 1598 self.writeNextBlock()
1599 1599 # self.getDataHeader()
1600 1600
1601 1601 return 1
1602 1602
1603 1603 def __getProcessFlags(self):
1604 1604
1605 1605 processFlags = 0
1606 1606
1607 1607 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
1608 1608 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
1609 1609 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
1610 1610 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
1611 1611 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
1612 1612 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
1613 1613
1614 1614 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
1615 1615
1616 1616
1617 1617
1618 1618 datatypeValueList = [PROCFLAG.DATATYPE_CHAR,
1619 1619 PROCFLAG.DATATYPE_SHORT,
1620 1620 PROCFLAG.DATATYPE_LONG,
1621 1621 PROCFLAG.DATATYPE_INT64,
1622 1622 PROCFLAG.DATATYPE_FLOAT,
1623 1623 PROCFLAG.DATATYPE_DOUBLE]
1624 1624
1625 1625
1626 1626 for index in range(len(dtypeList)):
1627 1627 if self.dataOut.dtype == dtypeList[index]:
1628 1628 dtypeValue = datatypeValueList[index]
1629 1629 break
1630 1630
1631 1631 processFlags += dtypeValue
1632 1632
1633 1633 if self.dataOut.flagDecodeData:
1634 1634 processFlags += PROCFLAG.DECODE_DATA
1635 1635
1636 1636 if self.dataOut.flagDeflipData:
1637 1637 processFlags += PROCFLAG.DEFLIP_DATA
1638 1638
1639 1639 if self.dataOut.code != None:
1640 1640 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1641 1641
1642 1642 if self.dataOut.nCohInt > 1:
1643 1643 processFlags += PROCFLAG.COHERENT_INTEGRATION
1644 1644
1645 1645 return processFlags
1646 1646
1647 1647
1648 1648 def __getBlockSize(self):
1649 1649 '''
1650 1650 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
1651 1651 '''
1652 1652
1653 1653 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
1654 1654 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
1655 1655 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
1656 1656 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
1657 1657 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
1658 1658 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
1659 1659
1660 1660 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
1661 1661 datatypeValueList = [1,2,4,8,4,8]
1662 1662 for index in range(len(dtypeList)):
1663 1663 if self.dataOut.dtype == dtypeList[index]:
1664 1664 datatypeValue = datatypeValueList[index]
1665 1665 break
1666 1666
1667 1667 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.dataOut.nProfiles * datatypeValue * 2)
1668 1668
1669 1669 return blocksize
1670 1670
1671 1671 def getDataHeader(self):
1672 1672
1673 1673 """
1674 1674 Obtiene una copia del First Header
1675 1675
1676 1676 Affected:
1677 1677 self.systemHeaderObj
1678 1678 self.radarControllerHeaderObj
1679 1679 self.dtype
1680 1680
1681 1681 Return:
1682 1682 None
1683 1683 """
1684 1684
1685 1685 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
1686 1686 self.systemHeaderObj.nChannels = self.dataOut.nChannels
1687 1687 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
1688 1688
1689 1689 self.getBasicHeader()
1690 1690
1691 1691 processingHeaderSize = 40 # bytes
1692 1692 self.processingHeaderObj.dtype = 0 # Voltage
1693 1693 self.processingHeaderObj.blockSize = self.__getBlockSize()
1694 1694 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
1695 1695 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
1696 1696 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
1697 1697 self.processingHeaderObj.processFlags = self.__getProcessFlags()
1698 1698 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
1699 1699 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
1700 1700 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
1701 1701
1702 1702 if self.dataOut.code != None:
1703 1703 self.processingHeaderObj.code = self.dataOut.code
1704 1704 self.processingHeaderObj.nCode = self.dataOut.nCode
1705 1705 self.processingHeaderObj.nBaud = self.dataOut.nBaud
1706 1706 codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)
1707 1707 processingHeaderSize += codesize
1708 1708
1709 1709 if self.processingHeaderObj.nWindows != 0:
1710 1710 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
1711 1711 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
1712 1712 self.processingHeaderObj.nHeights = self.dataOut.nHeights
1713 1713 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
1714 1714 processingHeaderSize += 12
1715 1715
1716 1716 self.processingHeaderObj.size = processingHeaderSize
1717 1717
1718 1718 class SpectraReader(JRODataReader):
1719 1719 """
1720 1720 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
1721 1721 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
1722 1722 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
1723 1723
1724 1724 paresCanalesIguales * alturas * perfiles (Self Spectra)
1725 1725 paresCanalesDiferentes * alturas * perfiles (Cross Spectra)
1726 1726 canales * alturas (DC Channels)
1727 1727
1728 1728 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
1729 1729 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
1730 1730 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
1731 1731 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
1732 1732
1733 1733 Example:
1734 1734 dpath = "/home/myuser/data"
1735 1735
1736 1736 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
1737 1737
1738 1738 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
1739 1739
1740 1740 readerObj = SpectraReader()
1741 1741
1742 1742 readerObj.setup(dpath, startTime, endTime)
1743 1743
1744 1744 while(True):
1745 1745
1746 1746 readerObj.getData()
1747 1747
1748 1748 print readerObj.data_spc
1749 1749
1750 1750 print readerObj.data_cspc
1751 1751
1752 1752 print readerObj.data_dc
1753 1753
1754 1754 if readerObj.flagNoMoreFiles:
1755 1755 break
1756 1756
1757 1757 """
1758 1758
1759 1759 pts2read_SelfSpectra = 0
1760 1760
1761 1761 pts2read_CrossSpectra = 0
1762 1762
1763 1763 pts2read_DCchannels = 0
1764 1764
1765 1765 ext = ".pdata"
1766 1766
1767 1767 optchar = "P"
1768 1768
1769 1769 dataOut = None
1770 1770
1771 1771 nRdChannels = None
1772 1772
1773 1773 nRdPairs = None
1774 1774
1775 1775 rdPairList = []
1776 1776
1777 1777
1778 1778 def __init__(self):
1779 1779 """
1780 1780 Inicializador de la clase SpectraReader para la lectura de datos de espectros.
1781 1781
1782 1782 Inputs:
1783 1783 dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para
1784 1784 almacenar un perfil de datos cada vez que se haga un requerimiento
1785 1785 (getData). El perfil sera obtenido a partir del buffer de datos,
1786 1786 si el buffer esta vacio se hara un nuevo proceso de lectura de un
1787 1787 bloque de datos.
1788 1788 Si este parametro no es pasado se creara uno internamente.
1789 1789
1790 1790 Affected:
1791 1791 self.dataOut
1792 1792
1793 1793 Return : None
1794 1794 """
1795 1795
1796 1796 self.isConfig = False
1797 1797
1798 1798 self.pts2read_SelfSpectra = 0
1799 1799
1800 1800 self.pts2read_CrossSpectra = 0
1801 1801
1802 1802 self.pts2read_DCchannels = 0
1803 1803
1804 1804 self.datablock = None
1805 1805
1806 1806 self.utc = None
1807 1807
1808 1808 self.ext = ".pdata"
1809 1809
1810 1810 self.optchar = "P"
1811 1811
1812 1812 self.basicHeaderObj = BasicHeader(LOCALTIME)
1813 1813
1814 1814 self.systemHeaderObj = SystemHeader()
1815 1815
1816 1816 self.radarControllerHeaderObj = RadarControllerHeader()
1817 1817
1818 1818 self.processingHeaderObj = ProcessingHeader()
1819 1819
1820 1820 self.online = 0
1821 1821
1822 1822 self.fp = None
1823 1823
1824 1824 self.idFile = None
1825 1825
1826 1826 self.dtype = None
1827 1827
1828 1828 self.fileSizeByHeader = None
1829 1829
1830 1830 self.filenameList = []
1831 1831
1832 1832 self.filename = None
1833 1833
1834 1834 self.fileSize = None
1835 1835
1836 1836 self.firstHeaderSize = 0
1837 1837
1838 1838 self.basicHeaderSize = 24
1839 1839
1840 1840 self.pathList = []
1841 1841
1842 1842 self.lastUTTime = 0
1843 1843
1844 1844 self.maxTimeStep = 30
1845 1845
1846 1846 self.flagNoMoreFiles = 0
1847 1847
1848 1848 self.set = 0
1849 1849
1850 1850 self.path = None
1851 1851
1852 1852 self.delay = 3 #seconds
1853 1853
1854 1854 self.nTries = 3 #quantity tries
1855 1855
1856 1856 self.nFiles = 3 #number of files for searching
1857 1857
1858 1858 self.nReadBlocks = 0
1859 1859
1860 1860 self.flagIsNewFile = 1
1861 1861
1862 1862 self.ippSeconds = 0
1863 1863
1864 1864 self.flagTimeBlock = 0
1865 1865
1866 1866 self.flagIsNewBlock = 0
1867 1867
1868 1868 self.nTotalBlocks = 0
1869 1869
1870 1870 self.blocksize = 0
1871 1871
1872 1872 self.dataOut = self.createObjByDefault()
1873 1873
1874 1874
1875 1875 def createObjByDefault(self):
1876 1876
1877 1877 dataObj = Spectra()
1878 1878
1879 1879 return dataObj
1880 1880
1881 1881 def __hasNotDataInBuffer(self):
1882 1882 return 1
1883 1883
1884 1884
1885 1885 def getBlockDimension(self):
1886 1886 """
1887 1887 Obtiene la cantidad de puntos a leer por cada bloque de datos
1888 1888
1889 1889 Affected:
1890 1890 self.nRdChannels
1891 1891 self.nRdPairs
1892 1892 self.pts2read_SelfSpectra
1893 1893 self.pts2read_CrossSpectra
1894 1894 self.pts2read_DCchannels
1895 1895 self.blocksize
1896 1896 self.dataOut.nChannels
1897 1897 self.dataOut.nPairs
1898 1898
1899 1899 Return:
1900 1900 None
1901 1901 """
1902 1902 self.nRdChannels = 0
1903 1903 self.nRdPairs = 0
1904 1904 self.rdPairList = []
1905 1905
1906 1906 for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
1907 1907 if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
1908 1908 self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
1909 1909 else:
1910 1910 self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
1911 1911 self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
1912 1912
1913 1913 pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
1914 1914
1915 1915 self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
1916 1916 self.blocksize = self.pts2read_SelfSpectra
1917 1917
1918 1918 if self.processingHeaderObj.flag_cspc:
1919 1919 self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
1920 1920 self.blocksize += self.pts2read_CrossSpectra
1921 1921
1922 1922 if self.processingHeaderObj.flag_dc:
1923 1923 self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
1924 1924 self.blocksize += self.pts2read_DCchannels
1925 1925
1926 1926 # self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
1927 1927
1928 1928
1929 1929 def readBlock(self):
1930 1930 """
1931 1931 Lee el bloque de datos desde la posicion actual del puntero del archivo
1932 1932 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
1933 1933 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
1934 1934 es seteado a 0
1935 1935
1936 1936 Return: None
1937 1937
1938 1938 Variables afectadas:
1939 1939
1940 1940 self.flagIsNewFile
1941 1941 self.flagIsNewBlock
1942 1942 self.nTotalBlocks
1943 1943 self.data_spc
1944 1944 self.data_cspc
1945 1945 self.data_dc
1946 1946
1947 1947 Exceptions:
1948 1948 Si un bloque leido no es un bloque valido
1949 1949 """
1950 1950 blockOk_flag = False
1951 1951 fpointer = self.fp.tell()
1952 1952
1953 1953 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
1954 1954 spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
1955 1955
1956 1956 if self.processingHeaderObj.flag_cspc:
1957 1957 cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
1958 1958 cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
1959 1959
1960 1960 if self.processingHeaderObj.flag_dc:
1961 1961 dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
1962 1962 dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
1963 1963
1964 1964
1965 1965 if not(self.processingHeaderObj.shif_fft):
1966 1966 #desplaza a la derecha en el eje 2 determinadas posiciones
1967 1967 shift = int(self.processingHeaderObj.profilesPerBlock/2)
1968 1968 spc = numpy.roll( spc, shift , axis=2 )
1969 1969
1970 1970 if self.processingHeaderObj.flag_cspc:
1971 1971 #desplaza a la derecha en el eje 2 determinadas posiciones
1972 1972 cspc = numpy.roll( cspc, shift, axis=2 )
1973 1973
1974 1974 # self.processingHeaderObj.shif_fft = True
1975 1975
1976 1976 spc = numpy.transpose( spc, (0,2,1) )
1977 1977 self.data_spc = spc
1978 1978
1979 1979 if self.processingHeaderObj.flag_cspc:
1980 1980 cspc = numpy.transpose( cspc, (0,2,1) )
1981 1981 self.data_cspc = cspc['real'] + cspc['imag']*1j
1982 1982 else:
1983 1983 self.data_cspc = None
1984 1984
1985 1985 if self.processingHeaderObj.flag_dc:
1986 1986 self.data_dc = dc['real'] + dc['imag']*1j
1987 1987 else:
1988 1988 self.data_dc = None
1989 1989
1990 1990 self.flagIsNewFile = 0
1991 1991 self.flagIsNewBlock = 1
1992 1992
1993 1993 self.nTotalBlocks += 1
1994 1994 self.nReadBlocks += 1
1995 1995
1996 1996 return 1
1997 1997
1998 1998
1999 1999 def getData(self):
2000 2000 """
2001 2001 Copia el buffer de lectura a la clase "Spectra",
2002 2002 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
2003 2003 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
2004 2004
2005 2005 Return:
2006 2006 0 : Si no hay mas archivos disponibles
2007 2007 1 : Si hizo una buena copia del buffer
2008 2008
2009 2009 Affected:
2010 2010 self.dataOut
2011 2011
2012 2012 self.flagTimeBlock
2013 2013 self.flagIsNewBlock
2014 2014 """
2015 2015
2016 2016 if self.flagNoMoreFiles:
2017 2017 self.dataOut.flagNoData = True
2018 2018 print 'Process finished'
2019 2019 return 0
2020 2020
2021 2021 self.flagTimeBlock = 0
2022 2022 self.flagIsNewBlock = 0
2023 2023
2024 2024 if self.__hasNotDataInBuffer():
2025 2025
2026 2026 if not( self.readNextBlock() ):
2027 2027 self.dataOut.flagNoData = True
2028 2028 return 0
2029 2029
2030 2030 # self.updateDataHeader()
2031 2031
2032 2032 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
2033 2033
2034 2034 if self.data_dc == None:
2035 2035 self.dataOut.flagNoData = True
2036 2036 return 0
2037 2037
2038 2038 self.dataOut.data_spc = self.data_spc
2039 2039
2040 2040 self.dataOut.data_cspc = self.data_cspc
2041 2041
2042 2042 self.dataOut.data_dc = self.data_dc
2043 2043
2044 2044 self.dataOut.flagTimeBlock = self.flagTimeBlock
2045 2045
2046 2046 self.dataOut.flagNoData = False
2047 2047
2048 self.dataOut.dtype = numpy.dtype([('real','<f8'),('imag','<f8')])#self.dtype
2048 self.dataOut.dtype = self.dtype
2049 2049
2050 2050 # self.dataOut.nChannels = self.nRdChannels
2051 2051
2052 2052 self.dataOut.nPairs = self.nRdPairs
2053 2053
2054 2054 self.dataOut.pairsList = self.rdPairList
2055 2055
2056 2056 # self.dataOut.nHeights = self.processingHeaderObj.nHeights
2057 2057
2058 2058 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
2059 2059
2060 2060 self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
2061 2061
2062 2062 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
2063 2063
2064 2064 self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
2065 2065
2066 2066 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
2067 2067
2068 2068 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
2069 2069
2070 2070 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
2071 2071
2072 2072 # self.dataOut.channelIndexList = range(self.systemHeaderObj.nChannels)
2073 2073
2074 2074 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000.#+ self.profileIndex * self.ippSeconds
2075 2075
2076 2076 self.dataOut.ippSeconds = self.ippSeconds
2077 2077
2078 2078 self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints
2079 2079
2080 2080 # self.profileIndex += 1
2081 2081
2082 2082 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
2083 2083
2084 2084 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
2085 2085
2086 2086 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
2087 2087
2088 2088 self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
2089 2089
2090 2090 self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip
2091 2091
2092 2092 if self.processingHeaderObj.code != None:
2093 2093
2094 2094 self.dataOut.nCode = self.processingHeaderObj.nCode
2095 2095
2096 2096 self.dataOut.nBaud = self.processingHeaderObj.nBaud
2097 2097
2098 2098 self.dataOut.code = self.processingHeaderObj.code
2099 2099
2100 2100 self.dataOut.flagDecodeData = True
2101 2101
2102 2102 return self.dataOut.data_spc
2103 2103
2104 2104
2105 2105 class SpectraWriter(JRODataWriter):
2106 2106
2107 2107 """
2108 2108 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
2109 2109 de los datos siempre se realiza por bloques.
2110 2110 """
2111 2111
2112 2112 ext = ".pdata"
2113 2113
2114 2114 optchar = "P"
2115 2115
2116 2116 shape_spc_Buffer = None
2117 2117
2118 2118 shape_cspc_Buffer = None
2119 2119
2120 2120 shape_dc_Buffer = None
2121 2121
2122 2122 data_spc = None
2123 2123
2124 2124 data_cspc = None
2125 2125
2126 2126 data_dc = None
2127 2127
2128 2128 # dataOut = None
2129 2129
2130 2130 def __init__(self):
2131 2131 """
2132 2132 Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
2133 2133
2134 2134 Affected:
2135 2135 self.dataOut
2136 2136 self.basicHeaderObj
2137 2137 self.systemHeaderObj
2138 2138 self.radarControllerHeaderObj
2139 2139 self.processingHeaderObj
2140 2140
2141 2141 Return: None
2142 2142 """
2143 2143
2144 2144 self.isConfig = False
2145 2145
2146 2146 self.nTotalBlocks = 0
2147 2147
2148 2148 self.data_spc = None
2149 2149
2150 2150 self.data_cspc = None
2151 2151
2152 2152 self.data_dc = None
2153 2153
2154 2154 self.fp = None
2155 2155
2156 2156 self.flagIsNewFile = 1
2157 2157
2158 2158 self.nTotalBlocks = 0
2159 2159
2160 2160 self.flagIsNewBlock = 0
2161 2161
2162 2162 self.setFile = None
2163 2163
2164 2164 self.dtype = None
2165 2165
2166 2166 self.path = None
2167 2167
2168 2168 self.noMoreFiles = 0
2169 2169
2170 2170 self.filename = None
2171 2171
2172 2172 self.basicHeaderObj = BasicHeader(LOCALTIME)
2173 2173
2174 2174 self.systemHeaderObj = SystemHeader()
2175 2175
2176 2176 self.radarControllerHeaderObj = RadarControllerHeader()
2177 2177
2178 2178 self.processingHeaderObj = ProcessingHeader()
2179 2179
2180 2180
2181 2181 def hasAllDataInBuffer(self):
2182 2182 return 1
2183 2183
2184 2184
2185 2185 def setBlockDimension(self):
2186 2186 """
2187 2187 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
2188 2188
2189 2189 Affected:
2190 2190 self.shape_spc_Buffer
2191 2191 self.shape_cspc_Buffer
2192 2192 self.shape_dc_Buffer
2193 2193
2194 2194 Return: None
2195 2195 """
2196 2196 self.shape_spc_Buffer = (self.dataOut.nChannels,
2197 2197 self.processingHeaderObj.nHeights,
2198 2198 self.processingHeaderObj.profilesPerBlock)
2199 2199
2200 2200 self.shape_cspc_Buffer = (self.dataOut.nPairs,
2201 2201 self.processingHeaderObj.nHeights,
2202 2202 self.processingHeaderObj.profilesPerBlock)
2203 2203
2204 2204 self.shape_dc_Buffer = (self.dataOut.nChannels,
2205 2205 self.processingHeaderObj.nHeights)
2206 2206
2207 2207
2208 2208 def writeBlock(self):
2209 2209 """
2210 2210 Escribe el buffer en el file designado
2211 2211
2212 2212 Affected:
2213 2213 self.data_spc
2214 2214 self.data_cspc
2215 2215 self.data_dc
2216 2216 self.flagIsNewFile
2217 2217 self.flagIsNewBlock
2218 2218 self.nTotalBlocks
2219 2219 self.nWriteBlocks
2220 2220
2221 2221 Return: None
2222 2222 """
2223 2223
2224 2224 spc = numpy.transpose( self.data_spc, (0,2,1) )
2225 2225 if not( self.processingHeaderObj.shif_fft ):
2226 2226 spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
2227 2227 data = spc.reshape((-1))
2228 data = data.astype(self.dtype[0])
2228 2229 data.tofile(self.fp)
2229 2230
2230 2231 if self.data_cspc != None:
2231 2232 data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )
2232 2233 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
2233 2234 if not( self.processingHeaderObj.shif_fft ):
2234 2235 cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
2235 2236 data['real'] = cspc.real
2236 2237 data['imag'] = cspc.imag
2237 2238 data = data.reshape((-1))
2238 2239 data.tofile(self.fp)
2239 2240
2240 2241 if self.data_dc != None:
2241 2242 data = numpy.zeros( self.shape_dc_Buffer, self.dtype )
2242 2243 dc = self.data_dc
2243 2244 data['real'] = dc.real
2244 2245 data['imag'] = dc.imag
2245 2246 data = data.reshape((-1))
2246 2247 data.tofile(self.fp)
2247 2248
2248 2249 self.data_spc.fill(0)
2249 2250 self.data_dc.fill(0)
2250 2251 if self.data_cspc != None:
2251 2252 self.data_cspc.fill(0)
2252 2253
2253 2254 self.flagIsNewFile = 0
2254 2255 self.flagIsNewBlock = 1
2255 2256 self.nTotalBlocks += 1
2256 2257 self.nWriteBlocks += 1
2257 2258 self.blockIndex += 1
2258 2259
2259 2260
2260 2261 def putData(self):
2261 2262 """
2262 2263 Setea un bloque de datos y luego los escribe en un file
2263 2264
2264 2265 Affected:
2265 2266 self.data_spc
2266 2267 self.data_cspc
2267 2268 self.data_dc
2268 2269
2269 2270 Return:
2270 2271 0 : Si no hay data o no hay mas files que puedan escribirse
2271 2272 1 : Si se escribio la data de un bloque en un file
2272 2273 """
2273 2274
2274 2275 if self.dataOut.flagNoData:
2275 2276 return 0
2276 2277
2277 2278 self.flagIsNewBlock = 0
2278 2279
2279 2280 if self.dataOut.flagTimeBlock:
2280 2281 self.data_spc.fill(0)
2281 2282 self.data_cspc.fill(0)
2282 2283 self.data_dc.fill(0)
2283 2284 self.setNextFile()
2284 2285
2285 2286 if self.flagIsNewFile == 0:
2286 2287 self.getBasicHeader()
2287 2288
2288 2289 self.data_spc = self.dataOut.data_spc.copy()
2289 2290 self.data_cspc = self.dataOut.data_cspc.copy()
2290 2291 self.data_dc = self.dataOut.data_dc.copy()
2291 2292
2292 2293 # #self.processingHeaderObj.dataBlocksPerFile)
2293 2294 if self.hasAllDataInBuffer():
2294 2295 # self.getDataHeader()
2295 2296 self.writeNextBlock()
2296 2297
2297 2298 return 1
2298 2299
2299 2300
2300 2301 def __getProcessFlags(self):
2301 2302
2302 2303 processFlags = 0
2303 2304
2304 2305 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
2305 2306 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
2306 2307 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
2307 2308 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
2308 2309 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
2309 2310 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
2310 2311
2311 2312 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
2312 2313
2313 2314
2314 2315
2315 2316 datatypeValueList = [PROCFLAG.DATATYPE_CHAR,
2316 2317 PROCFLAG.DATATYPE_SHORT,
2317 2318 PROCFLAG.DATATYPE_LONG,
2318 2319 PROCFLAG.DATATYPE_INT64,
2319 2320 PROCFLAG.DATATYPE_FLOAT,
2320 2321 PROCFLAG.DATATYPE_DOUBLE]
2321 2322
2322 2323
2323 2324 for index in range(len(dtypeList)):
2324 2325 if self.dataOut.dtype == dtypeList[index]:
2325 2326 dtypeValue = datatypeValueList[index]
2326 2327 break
2327 2328
2328 2329 processFlags += dtypeValue
2329 2330
2330 2331 if self.dataOut.flagDecodeData:
2331 2332 processFlags += PROCFLAG.DECODE_DATA
2332 2333
2333 2334 if self.dataOut.flagDeflipData:
2334 2335 processFlags += PROCFLAG.DEFLIP_DATA
2335 2336
2336 2337 if self.dataOut.code != None:
2337 2338 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
2338 2339
2339 2340 if self.dataOut.nIncohInt > 1:
2340 2341 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
2341 2342
2342 2343 if self.dataOut.data_dc != None:
2343 2344 processFlags += PROCFLAG.SAVE_CHANNELS_DC
2344 2345
2345 2346 return processFlags
2346 2347
2347 2348
2348 2349 def __getBlockSize(self):
2349 2350 '''
2350 2351 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
2351 2352 '''
2352 2353
2353 2354 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
2354 2355 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
2355 2356 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
2356 2357 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
2357 2358 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
2358 2359 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
2359 2360
2360 2361 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
2361 2362 datatypeValueList = [1,2,4,8,4,8]
2362 2363 for index in range(len(dtypeList)):
2363 2364 if self.dataOut.dtype == dtypeList[index]:
2364 2365 datatypeValue = datatypeValueList[index]
2365 2366 break
2366 2367
2367 2368
2368 2369 pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
2369 2370
2370 2371 pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
2371 2372 blocksize = (pts2write_SelfSpectra*datatypeValue)
2372 2373
2373 2374 if self.dataOut.data_cspc != None:
2374 2375 pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
2375 2376 blocksize += (pts2write_CrossSpectra*datatypeValue*2)
2376 2377
2377 2378 if self.dataOut.data_dc != None:
2378 2379 pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
2379 2380 blocksize += (pts2write_DCchannels*datatypeValue*2)
2380 2381
2381 2382 blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
2382 2383
2383 2384 return blocksize
2384 2385
2385 2386 def getDataHeader(self):
2386 2387
2387 2388 """
2388 2389 Obtiene una copia del First Header
2389 2390
2390 2391 Affected:
2391 2392 self.systemHeaderObj
2392 2393 self.radarControllerHeaderObj
2393 2394 self.dtype
2394 2395
2395 2396 Return:
2396 2397 None
2397 2398 """
2398 2399
2399 2400 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
2400 2401 self.systemHeaderObj.nChannels = self.dataOut.nChannels
2401 2402 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
2402 2403
2403 2404 self.getBasicHeader()
2404 2405
2405 2406 processingHeaderSize = 40 # bytes
2406 2407 self.processingHeaderObj.dtype = 0 # Voltage
2407 2408 self.processingHeaderObj.blockSize = self.__getBlockSize()
2408 2409 self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
2409 2410 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
2410 2411 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
2411 2412 self.processingHeaderObj.processFlags = self.__getProcessFlags()
2412 2413 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
2413 2414 self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
2414 2415 self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
2415 2416
2416 2417 if self.processingHeaderObj.totalSpectra > 0:
2417 2418 channelList = []
2418 2419 for channel in range(self.dataOut.nChannels):
2419 2420 channelList.append(channel)
2420 2421 channelList.append(channel)
2421 2422
2422 2423 pairsList = []
2423 2424 for pair in self.dataOut.pairsList:
2424 2425 pairsList.append(pair[0])
2425 2426 pairsList.append(pair[1])
2426 2427 spectraComb = channelList + pairsList
2427 2428 spectraComb = numpy.array(spectraComb,dtype="u1")
2428 2429 self.processingHeaderObj.spectraComb = spectraComb
2429 2430 sizeOfSpcComb = len(spectraComb)
2430 2431 processingHeaderSize += sizeOfSpcComb
2431 2432
2432 2433 if self.dataOut.code != None:
2433 2434 self.processingHeaderObj.code = self.dataOut.code
2434 2435 self.processingHeaderObj.nCode = self.dataOut.nCode
2435 2436 self.processingHeaderObj.nBaud = self.dataOut.nBaud
2436 2437 nCodeSize = 4 # bytes
2437 2438 nBaudSize = 4 # bytes
2438 2439 codeSize = 4 # bytes
2439 2440 sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)
2440 2441 processingHeaderSize += sizeOfCode
2441 2442
2442 2443 if self.processingHeaderObj.nWindows != 0:
2443 2444 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
2444 2445 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
2445 2446 self.processingHeaderObj.nHeights = self.dataOut.nHeights
2446 2447 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
2447 2448 sizeOfFirstHeight = 4
2448 2449 sizeOfdeltaHeight = 4
2449 2450 sizeOfnHeights = 4
2450 2451 sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows
2451 2452 processingHeaderSize += sizeOfWindows
2452 2453
2453 2454 self.processingHeaderObj.size = processingHeaderSize
2454 2455
2455 2456 class SpectraHeisWriter():
2456 2457
2457 2458 i=0
2458 2459
2459 2460 def __init__(self, dataOut):
2460 2461
2461 2462 self.wrObj = FITS()
2462 2463 self.dataOut = dataOut
2463 2464
2464 2465 def isNumber(str):
2465 2466 """
2466 2467 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
2467 2468
2468 2469 Excepciones:
2469 2470 Si un determinado string no puede ser convertido a numero
2470 2471 Input:
2471 2472 str, string al cual se le analiza para determinar si convertible a un numero o no
2472 2473
2473 2474 Return:
2474 2475 True : si el string es uno numerico
2475 2476 False : no es un string numerico
2476 2477 """
2477 2478 try:
2478 2479 float( str )
2479 2480 return True
2480 2481 except:
2481 2482 return False
2482 2483
2483 2484 def setup(self, wrpath,):
2484 2485
2485 2486 if not(os.path.exists(wrpath)):
2486 2487 os.mkdir(wrpath)
2487 2488
2488 2489 self.wrpath = wrpath
2489 2490 self.setFile = 0
2490 2491
2491 2492 def putData(self):
2492 2493 # self.wrObj.writeHeader(nChannels=self.dataOut.nChannels, nFFTPoints=self.dataOut.nFFTPoints)
2493 2494 #name = self.dataOut.utctime
2494 2495 name= time.localtime( self.dataOut.utctime)
2495 2496 ext=".fits"
2496 2497 #folder='D%4.4d%3.3d'%(name.tm_year,name.tm_yday)
2497 2498 subfolder = 'D%4.4d%3.3d' % (name.tm_year,name.tm_yday)
2498 2499
2499 2500 fullpath = os.path.join( self.wrpath, subfolder )
2500 2501 if not( os.path.exists(fullpath) ):
2501 2502 os.mkdir(fullpath)
2502 2503 self.setFile += 1
2503 2504 file = 'D%4.4d%3.3d%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
2504 2505
2505 2506 filename = os.path.join(self.wrpath,subfolder, file)
2506 2507
2507 2508 # print self.dataOut.ippSeconds
2508 2509 freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)/(2*self.dataOut.ippSeconds)
2509 2510
2510 2511 col1=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
2511 2512 col2=self.wrObj.writeData(name="P_Ch1",format=str(self.dataOut.nFFTPoints)+'E',data=10*numpy.log10(self.dataOut.data_spc[0,:]))
2512 2513 col3=self.wrObj.writeData(name="P_Ch2",format=str(self.dataOut.nFFTPoints)+'E',data=10*numpy.log10(self.dataOut.data_spc[1,:]))
2513 2514 col4=self.wrObj.writeData(name="P_Ch3",format=str(self.dataOut.nFFTPoints)+'E',data=10*numpy.log10(self.dataOut.data_spc[2,:]))
2514 2515 col5=self.wrObj.writeData(name="P_Ch4",format=str(self.dataOut.nFFTPoints)+'E',data=10*numpy.log10(self.dataOut.data_spc[3,:]))
2515 2516 col6=self.wrObj.writeData(name="P_Ch5",format=str(self.dataOut.nFFTPoints)+'E',data=10*numpy.log10(self.dataOut.data_spc[4,:]))
2516 2517 col7=self.wrObj.writeData(name="P_Ch6",format=str(self.dataOut.nFFTPoints)+'E',data=10*numpy.log10(self.dataOut.data_spc[5,:]))
2517 2518 col8=self.wrObj.writeData(name="P_Ch7",format=str(self.dataOut.nFFTPoints)+'E',data=10*numpy.log10(self.dataOut.data_spc[6,:]))
2518 2519 col9=self.wrObj.writeData(name="P_Ch8",format=str(self.dataOut.nFFTPoints)+'E',data=10*numpy.log10(self.dataOut.data_spc[7,:]))
2519 2520 #n=numpy.arange((100))
2520 2521 n=self.dataOut.data_spc[6,:]
2521 2522 a=self.wrObj.cFImage(n)
2522 2523 b=self.wrObj.Ctable(col1,col2,col3,col4,col5,col6,col7,col8,col9)
2523 2524 self.wrObj.CFile(a,b)
2524 2525 self.wrObj.wFile(filename)
2525 2526 return 1
2526 2527
2527 2528 class FITS:
2528 2529
2529 2530 name=None
2530 2531 format=None
2531 2532 array =None
2532 2533 data =None
2533 2534 thdulist=None
2534 2535
2535 2536 def __init__(self):
2536 2537
2537 2538 pass
2538 2539
2539 2540 def setColF(self,name,format,array):
2540 2541 self.name=name
2541 2542 self.format=format
2542 2543 self.array=array
2543 2544 a1=numpy.array([self.array],dtype=numpy.float32)
2544 2545 self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)
2545 2546 return self.col1
2546 2547
2547 2548 # def setColP(self,name,format,data):
2548 2549 # self.name=name
2549 2550 # self.format=format
2550 2551 # self.data=data
2551 2552 # a2=numpy.array([self.data],dtype=numpy.float32)
2552 2553 # self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
2553 2554 # return self.col2
2554 2555
2555 2556 def writeHeader(self,):
2556 2557 pass
2557 2558
2558 2559 def writeData(self,name,format,data):
2559 2560 self.name=name
2560 2561 self.format=format
2561 2562 self.data=data
2562 2563 a2=numpy.array([self.data],dtype=numpy.float32)
2563 2564 self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
2564 2565 return self.col2
2565 2566
2566 2567 def cFImage(self,n):
2567 2568 self.hdu= pyfits.PrimaryHDU(n)
2568 2569 return self.hdu
2569 2570
2570 2571 def Ctable(self,col1,col2,col3,col4,col5,col6,col7,col8,col9):
2571 2572 self.cols=pyfits.ColDefs( [col1,col2,col3,col4,col5,col6,col7,col8,col9])
2572 2573 self.tbhdu = pyfits.new_table(self.cols)
2573 2574 return self.tbhdu
2574 2575
2575 2576 def CFile(self,hdu,tbhdu):
2576 2577 self.thdulist=pyfits.HDUList([hdu,tbhdu])
2577 2578
2578 2579 def wFile(self,filename):
2579 2580 self.thdulist.writeto(filename) No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1281 +1,1282
1 1 '''
2 2
3 3 $Author: dsuarez $
4 4 $Id: Processor.py 1 2012-11-12 18:56:07Z dsuarez $
5 5 '''
6 6 import os
7 7 import numpy
8 8 import datetime
9 9 import time
10 10
11 11 from jrodata import *
12 12 from jrodataIO import *
13 13 from jroplot import *
14 14
15 15 class ProcessingUnit:
16 16
17 17 """
18 18 Esta es la clase base para el procesamiento de datos.
19 19
20 20 Contiene el metodo "call" para llamar operaciones. Las operaciones pueden ser:
21 21 - Metodos internos (callMethod)
22 22 - Objetos del tipo Operation (callObject). Antes de ser llamados, estos objetos
23 23 tienen que ser agreagados con el metodo "add".
24 24
25 25 """
26 26 # objeto de datos de entrada (Voltage, Spectra o Correlation)
27 27 dataIn = None
28 28
29 29 # objeto de datos de entrada (Voltage, Spectra o Correlation)
30 30 dataOut = None
31 31
32 32
33 33 objectDict = None
34 34
35 35 def __init__(self):
36 36
37 37 self.objectDict = {}
38 38
39 39 def init(self):
40 40
41 41 raise ValueError, "Not implemented"
42 42
43 43 def addOperation(self, object, objId):
44 44
45 45 """
46 46 Agrega el objeto "object" a la lista de objetos "self.objectList" y retorna el
47 47 identificador asociado a este objeto.
48 48
49 49 Input:
50 50
51 51 object : objeto de la clase "Operation"
52 52
53 53 Return:
54 54
55 55 objId : identificador del objeto, necesario para ejecutar la operacion
56 56 """
57 57
58 58 self.objectDict[objId] = object
59 59
60 60 return objId
61 61
62 62 def operation(self, **kwargs):
63 63
64 64 """
65 65 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
66 66 atributos del objeto dataOut
67 67
68 68 Input:
69 69
70 70 **kwargs : Diccionario de argumentos de la funcion a ejecutar
71 71 """
72 72
73 73 raise ValueError, "ImplementedError"
74 74
75 75 def callMethod(self, name, **kwargs):
76 76
77 77 """
78 78 Ejecuta el metodo con el nombre "name" y con argumentos **kwargs de la propia clase.
79 79
80 80 Input:
81 81 name : nombre del metodo a ejecutar
82 82
83 83 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
84 84
85 85 """
86 86 if name != 'run':
87 87
88 88 if name == 'init' and self.dataIn.isEmpty():
89 89 self.dataOut.flagNoData = True
90 90 return False
91 91
92 92 if name != 'init' and self.dataOut.isEmpty():
93 93 return False
94 94
95 95 methodToCall = getattr(self, name)
96 96
97 97 methodToCall(**kwargs)
98 98
99 99 if name != 'run':
100 100 return True
101 101
102 102 if self.dataOut.isEmpty():
103 103 return False
104 104
105 105 return True
106 106
107 107 def callObject(self, objId, **kwargs):
108 108
109 109 """
110 110 Ejecuta la operacion asociada al identificador del objeto "objId"
111 111
112 112 Input:
113 113
114 114 objId : identificador del objeto a ejecutar
115 115
116 116 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
117 117
118 118 Return:
119 119
120 120 None
121 121 """
122 122
123 123 if self.dataOut.isEmpty():
124 124 return False
125 125
126 126 object = self.objectDict[objId]
127 127
128 128 object.run(self.dataOut, **kwargs)
129 129
130 130 return True
131 131
132 132 def call(self, operationConf, **kwargs):
133 133
134 134 """
135 135 Return True si ejecuta la operacion "operationConf.name" con los
136 136 argumentos "**kwargs". False si la operacion no se ha ejecutado.
137 137 La operacion puede ser de dos tipos:
138 138
139 139 1. Un metodo propio de esta clase:
140 140
141 141 operation.type = "self"
142 142
143 143 2. El metodo "run" de un objeto del tipo Operation o de un derivado de ella:
144 144 operation.type = "other".
145 145
146 146 Este objeto de tipo Operation debe de haber sido agregado antes con el metodo:
147 147 "addOperation" e identificado con el operation.id
148 148
149 149
150 150 con el id de la operacion.
151 151
152 152 Input:
153 153
154 154 Operation : Objeto del tipo operacion con los atributos: name, type y id.
155 155
156 156 """
157 157
158 158 if operationConf.type == 'self':
159 159 sts = self.callMethod(operationConf.name, **kwargs)
160 160
161 161 if operationConf.type == 'other':
162 162 sts = self.callObject(operationConf.id, **kwargs)
163 163
164 164 return sts
165 165
166 166 def setInput(self, dataIn):
167 167
168 168 self.dataIn = dataIn
169 169
170 170 def getOutput(self):
171 171
172 172 return self.dataOut
173 173
174 174 class Operation():
175 175
176 176 """
177 177 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
178 178 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
179 179 acumulacion dentro de esta clase
180 180
181 181 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
182 182
183 183 """
184 184
185 185 __buffer = None
186 186 __isConfig = False
187 187
188 188 def __init__(self):
189 189
190 190 pass
191 191
192 192 def run(self, dataIn, **kwargs):
193 193
194 194 """
195 195 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los atributos del objeto dataIn.
196 196
197 197 Input:
198 198
199 199 dataIn : objeto del tipo JROData
200 200
201 201 Return:
202 202
203 203 None
204 204
205 205 Affected:
206 206 __buffer : buffer de recepcion de datos.
207 207
208 208 """
209 209
210 210 raise ValueError, "ImplementedError"
211 211
212 212 class VoltageProc(ProcessingUnit):
213 213
214 214
215 215 def __init__(self):
216 216
217 217 self.objectDict = {}
218 218 self.dataOut = Voltage()
219 219 self.flip = 1
220 220
221 221 def init(self):
222 222
223 223 self.dataOut.copy(self.dataIn)
224 224 # No necesita copiar en cada init() los atributos de dataIn
225 225 # la copia deberia hacerse por cada nuevo bloque de datos
226 226
227 227 def selectChannels(self, channelList):
228 228
229 229 channelIndexList = []
230 230
231 231 for channel in channelList:
232 232 index = self.dataOut.channelList.index(channel)
233 233 channelIndexList.append(index)
234 234
235 235 self.selectChannelsByIndex(channelIndexList)
236 236
237 237 def selectChannelsByIndex(self, channelIndexList):
238 238 """
239 239 Selecciona un bloque de datos en base a canales segun el channelIndexList
240 240
241 241 Input:
242 242 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
243 243
244 244 Affected:
245 245 self.dataOut.data
246 246 self.dataOut.channelIndexList
247 247 self.dataOut.nChannels
248 248 self.dataOut.m_ProcessingHeader.totalSpectra
249 249 self.dataOut.systemHeaderObj.numChannels
250 250 self.dataOut.m_ProcessingHeader.blockSize
251 251
252 252 Return:
253 253 None
254 254 """
255 255
256 256 for channelIndex in channelIndexList:
257 257 if channelIndex not in self.dataOut.channelIndexList:
258 258 print channelIndexList
259 259 raise ValueError, "The value %d in channelIndexList is not valid" %channelIndex
260 260
261 261 nChannels = len(channelIndexList)
262 262
263 263 data = self.dataOut.data[channelIndexList,:]
264 264
265 265 self.dataOut.data = data
266 266 self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
267 267 # self.dataOut.nChannels = nChannels
268 268
269 269 return 1
270 270
271 271 def selectHeights(self, minHei, maxHei):
272 272 """
273 273 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
274 274 minHei <= height <= maxHei
275 275
276 276 Input:
277 277 minHei : valor minimo de altura a considerar
278 278 maxHei : valor maximo de altura a considerar
279 279
280 280 Affected:
281 281 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
282 282
283 283 Return:
284 284 1 si el metodo se ejecuto con exito caso contrario devuelve 0
285 285 """
286 286 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
287 287 raise ValueError, "some value in (%d,%d) is not valid" % (minHei, maxHei)
288 288
289 289 if (maxHei > self.dataOut.heightList[-1]):
290 290 maxHei = self.dataOut.heightList[-1]
291 291 # raise ValueError, "some value in (%d,%d) is not valid" % (minHei, maxHei)
292 292
293 293 minIndex = 0
294 294 maxIndex = 0
295 295 heights = self.dataOut.heightList
296 296
297 297 inda = numpy.where(heights >= minHei)
298 298 indb = numpy.where(heights <= maxHei)
299 299
300 300 try:
301 301 minIndex = inda[0][0]
302 302 except:
303 303 minIndex = 0
304 304
305 305 try:
306 306 maxIndex = indb[0][-1]
307 307 except:
308 308 maxIndex = len(heights)
309 309
310 310 self.selectHeightsByIndex(minIndex, maxIndex)
311 311
312 312 return 1
313 313
314 314
315 315 def selectHeightsByIndex(self, minIndex, maxIndex):
316 316 """
317 317 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
318 318 minIndex <= index <= maxIndex
319 319
320 320 Input:
321 321 minIndex : valor de indice minimo de altura a considerar
322 322 maxIndex : valor de indice maximo de altura a considerar
323 323
324 324 Affected:
325 325 self.dataOut.data
326 326 self.dataOut.heightList
327 327
328 328 Return:
329 329 1 si el metodo se ejecuto con exito caso contrario devuelve 0
330 330 """
331 331
332 332 if (minIndex < 0) or (minIndex > maxIndex):
333 333 raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
334 334
335 335 if (maxIndex >= self.dataOut.nHeights):
336 336 maxIndex = self.dataOut.nHeights-1
337 337 # raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
338 338
339 339 nHeights = maxIndex - minIndex + 1
340 340
341 341 #voltage
342 342 data = self.dataOut.data[:,minIndex:maxIndex+1]
343 343
344 344 firstHeight = self.dataOut.heightList[minIndex]
345 345
346 346 self.dataOut.data = data
347 347 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex+1]
348 348
349 349 return 1
350 350
351 351
352 352 def filterByHeights(self, window):
353 353 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
354 354
355 355 if window == None:
356 356 window = self.dataOut.radarControllerHeaderObj.txA / deltaHeight
357 357
358 358 newdelta = deltaHeight * window
359 359 r = self.dataOut.data.shape[1] % window
360 360 buffer = self.dataOut.data[:,0:self.dataOut.data.shape[1]-r]
361 361 buffer = buffer.reshape(self.dataOut.data.shape[0],self.dataOut.data.shape[1]/window,window)
362 362 buffer = numpy.sum(buffer,2)
363 363 self.dataOut.data = buffer
364 364 self.dataOut.heightList = numpy.arange(self.dataOut.heightList[0],newdelta*self.dataOut.nHeights/window-newdelta,newdelta)
365 365 self.dataOut.windowOfFilter = window
366 366
367 367 def deFlip(self):
368 368 self.dataOut.data *= self.flip
369 369 self.flip *= -1.
370 370
371 371
372 372 class CohInt(Operation):
373 373
374 374 __isConfig = False
375 375
376 376 __profIndex = 0
377 377 __withOverapping = False
378 378
379 379 __byTime = False
380 380 __initime = None
381 381 __lastdatatime = None
382 382 __integrationtime = None
383 383
384 384 __buffer = None
385 385
386 386 __dataReady = False
387 387
388 388 n = None
389 389
390 390
391 391 def __init__(self):
392 392
393 393 self.__isConfig = False
394 394
395 395 def setup(self, n=None, timeInterval=None, overlapping=False):
396 396 """
397 397 Set the parameters of the integration class.
398 398
399 399 Inputs:
400 400
401 401 n : Number of coherent integrations
402 402 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
403 403 overlapping :
404 404
405 405 """
406 406
407 407 self.__initime = None
408 408 self.__lastdatatime = 0
409 409 self.__buffer = None
410 410 self.__dataReady = False
411 411
412 412
413 413 if n == None and timeInterval == None:
414 414 raise ValueError, "n or timeInterval should be specified ..."
415 415
416 416 if n != None:
417 417 self.n = n
418 418 self.__byTime = False
419 419 else:
420 420 self.__integrationtime = timeInterval * 60. #if (type(timeInterval)!=integer) -> change this line
421 421 self.n = 9999
422 422 self.__byTime = True
423 423
424 424 if overlapping:
425 425 self.__withOverapping = True
426 426 self.__buffer = None
427 427 else:
428 428 self.__withOverapping = False
429 429 self.__buffer = 0
430 430
431 431 self.__profIndex = 0
432 432
433 433 def putData(self, data):
434 434
435 435 """
436 436 Add a profile to the __buffer and increase in one the __profileIndex
437 437
438 438 """
439 439
440 440 if not self.__withOverapping:
441 441 self.__buffer += data.copy()
442 442 self.__profIndex += 1
443 443 return
444 444
445 445 #Overlapping data
446 446 nChannels, nHeis = data.shape
447 447 data = numpy.reshape(data, (1, nChannels, nHeis))
448 448
449 449 #If the buffer is empty then it takes the data value
450 450 if self.__buffer == None:
451 451 self.__buffer = data
452 452 self.__profIndex += 1
453 453 return
454 454
455 455 #If the buffer length is lower than n then stakcing the data value
456 456 if self.__profIndex < self.n:
457 457 self.__buffer = numpy.vstack((self.__buffer, data))
458 458 self.__profIndex += 1
459 459 return
460 460
461 461 #If the buffer length is equal to n then replacing the last buffer value with the data value
462 462 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
463 463 self.__buffer[self.n-1] = data
464 464 self.__profIndex = self.n
465 465 return
466 466
467 467
468 468 def pushData(self):
469 469 """
470 470 Return the sum of the last profiles and the profiles used in the sum.
471 471
472 472 Affected:
473 473
474 474 self.__profileIndex
475 475
476 476 """
477 477
478 478 if not self.__withOverapping:
479 479 data = self.__buffer
480 480 n = self.__profIndex
481 481
482 482 self.__buffer = 0
483 483 self.__profIndex = 0
484 484
485 485 return data, n
486 486
487 487 #Integration with Overlapping
488 488 data = numpy.sum(self.__buffer, axis=0)
489 489 n = self.__profIndex
490 490
491 491 return data, n
492 492
493 493 def byProfiles(self, data):
494 494
495 495 self.__dataReady = False
496 496 avgdata = None
497 497 n = None
498 498
499 499 self.putData(data)
500 500
501 501 if self.__profIndex == self.n:
502 502
503 503 avgdata, n = self.pushData()
504 504 self.__dataReady = True
505 505
506 506 return avgdata
507 507
508 508 def byTime(self, data, datatime):
509 509
510 510 self.__dataReady = False
511 511 avgdata = None
512 512 n = None
513 513
514 514 self.putData(data)
515 515
516 516 if (datatime - self.__initime) >= self.__integrationtime:
517 517 avgdata, n = self.pushData()
518 518 self.n = n
519 519 self.__dataReady = True
520 520
521 521 return avgdata
522 522
523 523 def integrate(self, data, datatime=None):
524 524
525 525 if self.__initime == None:
526 526 self.__initime = datatime
527 527
528 528 if self.__byTime:
529 529 avgdata = self.byTime(data, datatime)
530 530 else:
531 531 avgdata = self.byProfiles(data)
532 532
533 533
534 534 self.__lastdatatime = datatime
535 535
536 536 if avgdata == None:
537 537 return None, None
538 538
539 539 avgdatatime = self.__initime
540 540
541 541 deltatime = datatime -self.__lastdatatime
542 542
543 543 if not self.__withOverapping:
544 544 self.__initime = datatime
545 545 else:
546 546 self.__initime += deltatime
547 547
548 548 return avgdata, avgdatatime
549 549
550 550 def run(self, dataOut, **kwargs):
551 551
552 552 if not self.__isConfig:
553 553 self.setup(**kwargs)
554 554 self.__isConfig = True
555 555
556 556 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
557 557
558 558 # dataOut.timeInterval *= n
559 559 dataOut.flagNoData = True
560 560
561 561 if self.__dataReady:
562 562 dataOut.data = avgdata
563 563 dataOut.nCohInt *= self.n
564 564 dataOut.utctime = avgdatatime
565 565 dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
566 566 dataOut.flagNoData = False
567 567
568 568
569 569 class Decoder(Operation):
570 570
571 571 __isConfig = False
572 572 __profIndex = 0
573 573
574 574 code = None
575 575
576 576 nCode = None
577 577 nBaud = None
578 578
579 579 def __init__(self):
580 580
581 581 self.__isConfig = False
582 582
583 583 def setup(self, code):
584 584
585 585 self.__profIndex = 0
586 586
587 587 self.code = code
588 588
589 589 self.nCode = len(code)
590 590 self.nBaud = len(code[0])
591 591
592 592 def convolutionInFreq(self, data):
593 593
594 594 nchannel, ndata = data.shape
595 595 newcode = numpy.zeros(ndata)
596 596 newcode[0:self.nBaud] = self.code[self.__profIndex]
597 597
598 598 fft_data = numpy.fft.fft(data, axis=1)
599 599 fft_code = numpy.conj(numpy.fft.fft(newcode))
600 600 fft_code = fft_code.reshape(1,len(fft_code))
601 601
602 602 # conv = fft_data.copy()
603 603 # conv.fill(0)
604 604
605 605 conv = fft_data*fft_code
606 606
607 607 data = numpy.fft.ifft(conv,axis=1)
608 608
609 609 datadec = data[:,:-self.nBaud+1]
610 610 ndatadec = ndata - self.nBaud + 1
611 611
612 612 if self.__profIndex == self.nCode-1:
613 613 self.__profIndex = 0
614 614 return ndatadec, datadec
615 615
616 616 self.__profIndex += 1
617 617
618 618 return ndatadec, datadec
619 619
620 620
621 621 def convolutionInTime(self, data):
622 622
623 623 nchannel, ndata = data.shape
624 624 newcode = self.code[self.__profIndex]
625 625 ndatadec = ndata - self.nBaud + 1
626 626
627 627 datadec = numpy.zeros((nchannel, ndatadec))
628 628
629 629 for i in range(nchannel):
630 630 datadec[i,:] = numpy.correlate(data[i,:], newcode)
631 631
632 632 if self.__profIndex == self.nCode-1:
633 633 self.__profIndex = 0
634 634 return ndatadec, datadec
635 635
636 636 self.__profIndex += 1
637 637
638 638 return ndatadec, datadec
639 639
640 640 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0):
641 641
642 642 if not self.__isConfig:
643 643 if code != None:
644 644 code = numpy.array(code).reshape(nCode,nBaud)
645 645 if code == None:
646 646 code = dataOut.code
647 647
648 648 self.setup(code)
649 649 self.__isConfig = True
650 650
651 651 if mode == 0:
652 652 ndatadec, datadec = self.convolutionInFreq(dataOut.data)
653 653
654 654 if mode == 1:
655 655 print "This function is not implemented"
656 656 # ndatadec, datadec = self.convolutionInTime(dataOut.data)
657 657
658 658 dataOut.data = datadec
659 659
660 660 dataOut.heightList = dataOut.heightList[0:ndatadec]
661 661
662 662 dataOut.flagDecodeData = True #asumo q la data no esta decodificada
663 663
664 664 # dataOut.flagDeflipData = True #asumo q la data no esta sin flip
665 665
666 666
667 667 class SpectraProc(ProcessingUnit):
668 668
669 669 def __init__(self):
670 670
671 671 self.objectDict = {}
672 672 self.buffer = None
673 673 self.firstdatatime = None
674 674 self.profIndex = 0
675 675 self.dataOut = Spectra()
676 676
677 677 def __updateObjFromInput(self):
678 678
679 679 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
680 680 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
681 681 self.dataOut.channelList = self.dataIn.channelList
682 682 self.dataOut.heightList = self.dataIn.heightList
683 self.dataOut.dtype = self.dataIn.dtype
683 self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
684 684 # self.dataOut.nHeights = self.dataIn.nHeights
685 685 # self.dataOut.nChannels = self.dataIn.nChannels
686 686 self.dataOut.nBaud = self.dataIn.nBaud
687 687 self.dataOut.nCode = self.dataIn.nCode
688 688 self.dataOut.code = self.dataIn.code
689 689 self.dataOut.nProfiles = self.dataOut.nFFTPoints
690 690 # self.dataOut.channelIndexList = self.dataIn.channelIndexList
691 691 self.dataOut.flagTimeBlock = self.dataIn.flagTimeBlock
692 692 self.dataOut.utctime = self.firstdatatime
693 693 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
694 694 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip
695 695 self.dataOut.flagShiftFFT = self.dataIn.flagShiftFFT
696 696 self.dataOut.nCohInt = self.dataIn.nCohInt
697 697 self.dataOut.nIncohInt = 1
698 698 self.dataOut.ippSeconds = self.dataIn.ippSeconds
699 699 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
700 700
701 701 self.dataOut.timeInterval = self.dataIn.timeInterval*self.dataOut.nFFTPoints*self.dataOut.nIncohInt
702 702
703 703 def __getFft(self):
704 704 """
705 705 Convierte valores de Voltaje a Spectra
706 706
707 707 Affected:
708 708 self.dataOut.data_spc
709 709 self.dataOut.data_cspc
710 710 self.dataOut.data_dc
711 711 self.dataOut.heightList
712 712 self.profIndex
713 713 self.buffer
714 714 self.dataOut.flagNoData
715 715 """
716 716 fft_volt = numpy.fft.fft(self.buffer,axis=1)
717 fft_volt = fft_volt.astype(numpy.dtype('complex'))
717 718 dc = fft_volt[:,0,:]
718 719
719 720 #calculo de self-spectra
720 721 fft_volt = numpy.fft.fftshift(fft_volt,axes=(1,))
721 722 spc = fft_volt * numpy.conjugate(fft_volt)
722 723 spc = spc.real
723 724
724 725 blocksize = 0
725 726 blocksize += dc.size
726 727 blocksize += spc.size
727 728
728 729 cspc = None
729 730 pairIndex = 0
730 731 if self.dataOut.pairsList != None:
731 732 #calculo de cross-spectra
732 733 cspc = numpy.zeros((self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
733 734 for pair in self.dataOut.pairsList:
734 735 cspc[pairIndex,:,:] = fft_volt[pair[0],:,:] * numpy.conjugate(fft_volt[pair[1],:,:])
735 736 pairIndex += 1
736 737 blocksize += cspc.size
737 738
738 739 self.dataOut.data_spc = spc
739 740 self.dataOut.data_cspc = cspc
740 741 self.dataOut.data_dc = dc
741 742 self.dataOut.blockSize = blocksize
742 743
743 744 def init(self, nFFTPoints=None, pairsList=None):
744 745
745 746 self.dataOut.flagNoData = True
746 747
747 748 if self.dataIn.type == "Spectra":
748 749 self.dataOut.copy(self.dataIn)
749 750 return
750 751
751 752 if self.dataIn.type == "Voltage":
752 753
753 754 if nFFTPoints == None:
754 755 raise ValueError, "This SpectraProc.init() need nFFTPoints input variable"
755 756
756 757 if pairsList == None:
757 758 nPairs = 0
758 759 else:
759 760 nPairs = len(pairsList)
760 761
761 762 self.dataOut.nFFTPoints = nFFTPoints
762 763 self.dataOut.pairsList = pairsList
763 764 self.dataOut.nPairs = nPairs
764 765
765 766 if self.buffer == None:
766 767 self.buffer = numpy.zeros((self.dataIn.nChannels,
767 768 self.dataOut.nFFTPoints,
768 769 self.dataIn.nHeights),
769 770 dtype='complex')
770 771
771 772
772 773 self.buffer[:,self.profIndex,:] = self.dataIn.data.copy()
773 774 self.profIndex += 1
774 775
775 776 if self.firstdatatime == None:
776 777 self.firstdatatime = self.dataIn.utctime
777 778
778 779 if self.profIndex == self.dataOut.nFFTPoints:
779 780 self.__updateObjFromInput()
780 781 self.__getFft()
781 782
782 783 self.dataOut.flagNoData = False
783 784
784 785 self.buffer = None
785 786 self.firstdatatime = None
786 787 self.profIndex = 0
787 788
788 789 return
789 790
790 791 raise ValuError, "The type object %s is not valid"%(self.dataIn.type)
791 792
792 793 def selectChannels(self, channelList):
793 794
794 795 channelIndexList = []
795 796
796 797 for channel in channelList:
797 798 index = self.dataOut.channelList.index(channel)
798 799 channelIndexList.append(index)
799 800
800 801 self.selectChannelsByIndex(channelIndexList)
801 802
802 803 def selectChannelsByIndex(self, channelIndexList):
803 804 """
804 805 Selecciona un bloque de datos en base a canales segun el channelIndexList
805 806
806 807 Input:
807 808 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
808 809
809 810 Affected:
810 811 self.dataOut.data_spc
811 812 self.dataOut.channelIndexList
812 813 self.dataOut.nChannels
813 814
814 815 Return:
815 816 None
816 817 """
817 818
818 819 for channelIndex in channelIndexList:
819 820 if channelIndex not in self.dataOut.channelIndexList:
820 821 print channelIndexList
821 822 raise ValueError, "The value %d in channelIndexList is not valid" %channelIndex
822 823
823 824 nChannels = len(channelIndexList)
824 825
825 826 data_spc = self.dataOut.data_spc[channelIndexList,:]
826 827
827 828 self.dataOut.data_spc = data_spc
828 829 self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
829 830 # self.dataOut.nChannels = nChannels
830 831
831 832 return 1
832 833
833 834 def selectHeights(self, minHei, maxHei):
834 835 """
835 836 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
836 837 minHei <= height <= maxHei
837 838
838 839 Input:
839 840 minHei : valor minimo de altura a considerar
840 841 maxHei : valor maximo de altura a considerar
841 842
842 843 Affected:
843 844 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
844 845
845 846 Return:
846 847 1 si el metodo se ejecuto con exito caso contrario devuelve 0
847 848 """
848 849 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
849 850 raise ValueError, "some value in (%d,%d) is not valid" % (minHei, maxHei)
850 851
851 852 if (maxHei > self.dataOut.heightList[-1]):
852 853 maxHei = self.dataOut.heightList[-1]
853 854 # raise ValueError, "some value in (%d,%d) is not valid" % (minHei, maxHei)
854 855
855 856 minIndex = 0
856 857 maxIndex = 0
857 858 heights = self.dataOut.heightList
858 859
859 860 inda = numpy.where(heights >= minHei)
860 861 indb = numpy.where(heights <= maxHei)
861 862
862 863 try:
863 864 minIndex = inda[0][0]
864 865 except:
865 866 minIndex = 0
866 867
867 868 try:
868 869 maxIndex = indb[0][-1]
869 870 except:
870 871 maxIndex = len(heights)
871 872
872 873 self.selectHeightsByIndex(minIndex, maxIndex)
873 874
874 875 return 1
875 876
876 877
877 878 def selectHeightsByIndex(self, minIndex, maxIndex):
878 879 """
879 880 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
880 881 minIndex <= index <= maxIndex
881 882
882 883 Input:
883 884 minIndex : valor de indice minimo de altura a considerar
884 885 maxIndex : valor de indice maximo de altura a considerar
885 886
886 887 Affected:
887 888 self.dataOut.data_spc
888 889 self.dataOut.data_cspc
889 890 self.dataOut.data_dc
890 891 self.dataOut.heightList
891 892
892 893 Return:
893 894 1 si el metodo se ejecuto con exito caso contrario devuelve 0
894 895 """
895 896
896 897 if (minIndex < 0) or (minIndex > maxIndex):
897 898 raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
898 899
899 900 if (maxIndex >= self.dataOut.nHeights):
900 901 maxIndex = self.dataOut.nHeights-1
901 902 # raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
902 903
903 904 nHeights = maxIndex - minIndex + 1
904 905
905 906 #Spectra
906 907 data_spc = self.dataOut.data_spc[:,:,minIndex:maxIndex+1]
907 908
908 909 data_cspc = None
909 910 if self.dataOut.data_cspc != None:
910 911 data_cspc = self.dataOut.data_cspc[:,:,minIndex:maxIndex+1]
911 912
912 913 data_dc = None
913 914 if self.dataOut.data_dc != None:
914 915 data_dc = self.dataOut.data_dc[:,:,minIndex:maxIndex+1]
915 916
916 917 self.dataOut.data_spc = data_spc
917 918 self.dataOut.data_cspc = data_cspc
918 919 self.dataOut.data_dc = data_dc
919 920
920 921 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex+1]
921 922
922 923 return 1
923 924
924 925 def removeDC(self, mode = 1):
925 926
926 927 dc_index = 0
927 928 freq_index = numpy.array([-2,-1,1,2])
928 929 data_spc = self.dataOut.data_spc
929 930 data_cspc = self.dataOut.data_cspc
930 931 data_dc = self.dataOut.data_dc
931 932
932 933 if self.dataOut.flagShiftFFT:
933 934 dc_index += self.dataOut.nFFTPoints/2
934 935 freq_index += self.dataOut.nFFTPoints/2
935 936
936 937 if mode == 1:
937 938 data_spc[dc_index] = (data_spc[:,freq_index[1],:] + data_spc[:,freq_index[2],:])/2
938 939 if data_cspc != None:
939 940 data_cspc[dc_index] = (data_cspc[:,freq_index[1],:] + data_cspc[:,freq_index[2],:])/2
940 941 return 1
941 942
942 943 if mode == 2:
943 944 pass
944 945
945 946 if mode == 3:
946 947 pass
947 948
948 949 raise ValueError, "mode parameter has to be 1, 2 or 3"
949 950
950 951 def removeInterference(self):
951 952
952 953 pass
953 954
954 955
955 956 class IncohInt(Operation):
956 957
957 958
958 959 __profIndex = 0
959 960 __withOverapping = False
960 961
961 962 __byTime = False
962 963 __initime = None
963 964 __lastdatatime = None
964 965 __integrationtime = None
965 966
966 967 __buffer_spc = None
967 968 __buffer_cspc = None
968 969 __buffer_dc = None
969 970
970 971 __dataReady = False
971 972
972 973 n = None
973 974
974 975
975 976 def __init__(self):
976 977
977 978 self.__isConfig = False
978 979
979 980 def setup(self, n=None, timeInterval=None, overlapping=False):
980 981 """
981 982 Set the parameters of the integration class.
982 983
983 984 Inputs:
984 985
985 986 n : Number of coherent integrations
986 987 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
987 988 overlapping :
988 989
989 990 """
990 991
991 992 self.__initime = None
992 993 self.__lastdatatime = 0
993 994 self.__buffer_spc = None
994 995 self.__buffer_cspc = None
995 996 self.__buffer_dc = None
996 997 self.__dataReady = False
997 998
998 999
999 1000 if n == None and timeInterval == None:
1000 1001 raise ValueError, "n or timeInterval should be specified ..."
1001 1002
1002 1003 if n != None:
1003 1004 self.n = n
1004 1005 self.__byTime = False
1005 1006 else:
1006 1007 self.__integrationtime = timeInterval * 60. #if (type(timeInterval)!=integer) -> change this line
1007 1008 self.n = 9999
1008 1009 self.__byTime = True
1009 1010
1010 1011 if overlapping:
1011 1012 self.__withOverapping = True
1012 1013 else:
1013 1014 self.__withOverapping = False
1014 1015 self.__buffer_spc = 0
1015 1016 self.__buffer_cspc = 0
1016 1017 self.__buffer_dc = 0
1017 1018
1018 1019 self.__profIndex = 0
1019 1020
1020 1021 def putData(self, data_spc, data_cspc, data_dc):
1021 1022
1022 1023 """
1023 1024 Add a profile to the __buffer_spc and increase in one the __profileIndex
1024 1025
1025 1026 """
1026 1027
1027 1028 if not self.__withOverapping:
1028 1029 self.__buffer_spc += data_spc
1029 1030
1030 1031 if data_cspc == None:
1031 1032 self.__buffer_cspc = None
1032 1033 else:
1033 1034 self.__buffer_cspc += data_cspc
1034 1035
1035 1036 if data_dc == None:
1036 1037 self.__buffer_dc = None
1037 1038 else:
1038 1039 self.__buffer_dc += data_dc
1039 1040
1040 1041 self.__profIndex += 1
1041 1042 return
1042 1043
1043 1044 #Overlapping data
1044 1045 nChannels, nFFTPoints, nHeis = data_spc.shape
1045 1046 data_spc = numpy.reshape(data_spc, (1, nChannels, nFFTPoints, nHeis))
1046 1047 if data_cspc != None:
1047 1048 data_cspc = numpy.reshape(data_cspc, (1, -1, nFFTPoints, nHeis))
1048 1049 if data_dc != None:
1049 1050 data_dc = numpy.reshape(data_dc, (1, -1, nHeis))
1050 1051
1051 1052 #If the buffer is empty then it takes the data value
1052 1053 if self.__buffer_spc == None:
1053 1054 self.__buffer_spc = data_spc
1054 1055
1055 1056 if data_cspc == None:
1056 1057 self.__buffer_cspc = None
1057 1058 else:
1058 1059 self.__buffer_cspc += data_cspc
1059 1060
1060 1061 if data_dc == None:
1061 1062 self.__buffer_dc = None
1062 1063 else:
1063 1064 self.__buffer_dc += data_dc
1064 1065
1065 1066 self.__profIndex += 1
1066 1067 return
1067 1068
1068 1069 #If the buffer length is lower than n then stakcing the data value
1069 1070 if self.__profIndex < self.n:
1070 1071 self.__buffer_spc = numpy.vstack((self.__buffer_spc, data_spc))
1071 1072
1072 1073 if data_cspc != None:
1073 1074 self.__buffer_cspc = numpy.vstack((self.__buffer_cspc, data_cspc))
1074 1075
1075 1076 if data_dc != None:
1076 1077 self.__buffer_dc = numpy.vstack((self.__buffer_dc, data_dc))
1077 1078
1078 1079 self.__profIndex += 1
1079 1080 return
1080 1081
1081 1082 #If the buffer length is equal to n then replacing the last buffer value with the data value
1082 1083 self.__buffer_spc = numpy.roll(self.__buffer_spc, -1, axis=0)
1083 1084 self.__buffer_spc[self.n-1] = data_spc
1084 1085
1085 1086 if data_cspc != None:
1086 1087 self.__buffer_cspc = numpy.roll(self.__buffer_cspc, -1, axis=0)
1087 1088 self.__buffer_cspc[self.n-1] = data_cspc
1088 1089
1089 1090 if data_dc != None:
1090 1091 self.__buffer_dc = numpy.roll(self.__buffer_dc, -1, axis=0)
1091 1092 self.__buffer_dc[self.n-1] = data_dc
1092 1093
1093 1094 self.__profIndex = self.n
1094 1095 return
1095 1096
1096 1097
1097 1098 def pushData(self):
1098 1099 """
1099 1100 Return the sum of the last profiles and the profiles used in the sum.
1100 1101
1101 1102 Affected:
1102 1103
1103 1104 self.__profileIndex
1104 1105
1105 1106 """
1106 1107 data_spc = None
1107 1108 data_cspc = None
1108 1109 data_dc = None
1109 1110
1110 1111 if not self.__withOverapping:
1111 1112 data_spc = self.__buffer_spc
1112 1113 data_cspc = self.__buffer_cspc
1113 1114 data_dc = self.__buffer_dc
1114 1115
1115 1116 n = self.__profIndex
1116 1117
1117 1118 self.__buffer_spc = 0
1118 1119 self.__buffer_cspc = 0
1119 1120 self.__buffer_dc = 0
1120 1121 self.__profIndex = 0
1121 1122
1122 1123 return data_spc, data_cspc, data_dc, n
1123 1124
1124 1125 #Integration with Overlapping
1125 1126 data_spc = numpy.sum(self.__buffer_spc, axis=0)
1126 1127
1127 1128 if self.__buffer_cspc != None:
1128 1129 data_cspc = numpy.sum(self.__buffer_cspc, axis=0)
1129 1130
1130 1131 if self.__buffer_dc != None:
1131 1132 data_dc = numpy.sum(self.__buffer_dc, axis=0)
1132 1133
1133 1134 n = self.__profIndex
1134 1135
1135 1136 return data_spc, data_cspc, data_dc, n
1136 1137
1137 1138 def byProfiles(self, *args):
1138 1139
1139 1140 self.__dataReady = False
1140 1141 avgdata_spc = None
1141 1142 avgdata_cspc = None
1142 1143 avgdata_dc = None
1143 1144 n = None
1144 1145
1145 1146 self.putData(*args)
1146 1147
1147 1148 if self.__profIndex == self.n:
1148 1149
1149 1150 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1150 1151 self.__dataReady = True
1151 1152
1152 1153 return avgdata_spc, avgdata_cspc, avgdata_dc
1153 1154
1154 1155 def byTime(self, datatime, *args):
1155 1156
1156 1157 self.__dataReady = False
1157 1158 avgdata_spc = None
1158 1159 avgdata_cspc = None
1159 1160 avgdata_dc = None
1160 1161 n = None
1161 1162
1162 1163 self.putData(*args)
1163 1164
1164 1165 if (datatime - self.__initime) >= self.__integrationtime:
1165 1166 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1166 1167 self.n = n
1167 1168 self.__dataReady = True
1168 1169
1169 1170 return avgdata_spc, avgdata_cspc, avgdata_dc
1170 1171
1171 1172 def integrate(self, datatime, *args):
1172 1173
1173 1174 if self.__initime == None:
1174 1175 self.__initime = datatime
1175 1176
1176 1177 if self.__byTime:
1177 1178 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(datatime, *args)
1178 1179 else:
1179 1180 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1180 1181
1181 1182 self.__lastdatatime = datatime
1182 1183
1183 1184 if avgdata_spc == None:
1184 1185 return None, None, None, None
1185 1186
1186 1187 avgdatatime = self.__initime
1187 1188
1188 1189 deltatime = datatime -self.__lastdatatime
1189 1190
1190 1191 if not self.__withOverapping:
1191 1192 self.__initime = datatime
1192 1193 else:
1193 1194 self.__initime += deltatime
1194 1195
1195 1196 return avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc
1196 1197
1197 1198 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
1198 1199
1199 1200 if not self.__isConfig:
1200 1201 self.setup(n, timeInterval, overlapping)
1201 1202 self.__isConfig = True
1202 1203
1203 1204 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
1204 1205 dataOut.data_spc,
1205 1206 dataOut.data_cspc,
1206 1207 dataOut.data_dc)
1207 1208
1208 1209 # dataOut.timeInterval *= n
1209 1210 dataOut.flagNoData = True
1210 1211
1211 1212 if self.__dataReady:
1212 1213
1213 1214 dataOut.data_spc = avgdata_spc
1214 1215 dataOut.data_cspc = avgdata_cspc
1215 1216 dataOut.data_dc = avgdata_dc
1216 1217
1217 1218 dataOut.nIncohInt *= self.n
1218 1219 dataOut.utctime = avgdatatime
1219 1220 dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt * dataOut.nIncohInt * dataOut.nFFTPoints
1220 1221 dataOut.flagNoData = False
1221 1222
1222 1223 class ProfileSelector(Operation):
1223 1224
1224 1225 profileIndex = None
1225 1226 # Tamanho total de los perfiles
1226 1227 nProfiles = None
1227 1228
1228 1229 def __init__(self):
1229 1230
1230 1231 self.profileIndex = 0
1231 1232
1232 1233 def incIndex(self):
1233 1234 self.profileIndex += 1
1234 1235
1235 1236 if self.profileIndex >= self.nProfiles:
1236 1237 self.profileIndex = 0
1237 1238
1238 1239 def isProfileInRange(self, minIndex, maxIndex):
1239 1240
1240 1241 if self.profileIndex < minIndex:
1241 1242 return False
1242 1243
1243 1244 if self.profileIndex > maxIndex:
1244 1245 return False
1245 1246
1246 1247 return True
1247 1248
1248 1249 def isProfileInList(self, profileList):
1249 1250
1250 1251 if self.profileIndex not in profileList:
1251 1252 return False
1252 1253
1253 1254 return True
1254 1255
1255 1256 def run(self, dataOut, profileList=None, profileRangeList=None):
1256 1257
1257 1258 dataOut.flagNoData = True
1258 1259 self.nProfiles = dataOut.nProfiles
1259 1260
1260 1261 if profileList != None:
1261 1262 if self.isProfileInList(profileList):
1262 1263 dataOut.flagNoData = False
1263 1264
1264 1265 self.incIndex()
1265 1266 return 1
1266 1267
1267 1268
1268 1269 elif profileRangeList != None:
1269 1270 minIndex = profileRangeList[0]
1270 1271 maxIndex = profileRangeList[1]
1271 1272 if self.isProfileInRange(minIndex, maxIndex):
1272 1273 dataOut.flagNoData = False
1273 1274
1274 1275 self.incIndex()
1275 1276 return 1
1276 1277
1277 1278 else:
1278 1279 raise ValueError, "ProfileSelector needs profileList or profileRangeList"
1279 1280
1280 1281 return 0
1281 1282
General Comments 0
You need to be logged in to leave comments. Login now