##// END OF EJS Templates
schain
RSS
Se modifico las condiciones de Decoder, en cada iteracion se actualiza el codigo. Se corrigieron bugs en la lectura de codigo. Se detecto que no se guarda el codigo actualizado en el header de pdata, esta pendiente su revision.
Daniel Valdez -
r464:f8fb1296f4da
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,3418 +1,3418
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $
5 5 '''
6 6
7 7 import os, sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 from xml.etree.ElementTree import Element, SubElement, ElementTree
14 14 try:
15 15 import pyfits
16 16 except:
17 17 print "pyfits module has not been imported, it should be installed to save files in fits format"
18 18
19 19 from jrodata import *
20 20 from jroheaderIO import *
21 21 from jroprocessing import *
22 22
23 23 LOCALTIME = True #-18000
24 24
25 25 def isNumber(str):
26 26 """
27 27 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
28 28
29 29 Excepciones:
30 30 Si un determinado string no puede ser convertido a numero
31 31 Input:
32 32 str, string al cual se le analiza para determinar si convertible a un numero o no
33 33
34 34 Return:
35 35 True : si el string es uno numerico
36 36 False : no es un string numerico
37 37 """
38 38 try:
39 39 float( str )
40 40 return True
41 41 except:
42 42 return False
43 43
44 44 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
45 45 """
46 46 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47 47
48 48 Inputs:
49 49 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
50 50
51 51 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
52 52 segundos contados desde 01/01/1970.
53 53 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
54 54 segundos contados desde 01/01/1970.
55 55
56 56 Return:
57 57 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
58 58 fecha especificado, de lo contrario retorna False.
59 59
60 60 Excepciones:
61 61 Si el archivo no existe o no puede ser abierto
62 62 Si la cabecera no puede ser leida.
63 63
64 64 """
65 65 basicHeaderObj = BasicHeader(LOCALTIME)
66 66
67 67 try:
68 68 fp = open(filename,'rb')
69 69 except:
70 70 raise IOError, "The file %s can't be opened" %(filename)
71 71
72 72 sts = basicHeaderObj.read(fp)
73 73 fp.close()
74 74
75 75 if not(sts):
76 76 print "Skipping the file %s because it has not a valid header" %(filename)
77 77 return 0
78 78
79 79 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
80 80 return 0
81 81
82 82 return 1
83 83
84 84 def isFileinThisTime(filename, startTime, endTime):
85 85 """
86 86 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
87 87
88 88 Inputs:
89 89 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
90 90
91 91 startTime : tiempo inicial del rango seleccionado en formato datetime.time
92 92
93 93 endTime : tiempo final del rango seleccionado en formato datetime.time
94 94
95 95 Return:
96 96 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
97 97 fecha especificado, de lo contrario retorna False.
98 98
99 99 Excepciones:
100 100 Si el archivo no existe o no puede ser abierto
101 101 Si la cabecera no puede ser leida.
102 102
103 103 """
104 104
105 105
106 106 try:
107 107 fp = open(filename,'rb')
108 108 except:
109 109 raise IOError, "The file %s can't be opened" %(filename)
110 110
111 111 basicHeaderObj = BasicHeader(LOCALTIME)
112 112 sts = basicHeaderObj.read(fp)
113 113 fp.close()
114 114
115 115 thisDatetime = basicHeaderObj.datatime
116 116 thisTime = basicHeaderObj.datatime.time()
117 117
118 118 if not(sts):
119 119 print "Skipping the file %s because it has not a valid header" %(filename)
120 120 return None
121 121
122 122 if not ((startTime <= thisTime) and (endTime > thisTime)):
123 123 return None
124 124
125 125 return thisDatetime
126 126
127 127 def getFileFromSet(path,ext,set):
128 128 validFilelist = []
129 129 fileList = os.listdir(path)
130 130
131 131 # 0 1234 567 89A BCDE
132 132 # H YYYY DDD SSS .ext
133 133
134 134 for file in fileList:
135 135 try:
136 136 year = int(file[1:5])
137 137 doy = int(file[5:8])
138 138
139 139
140 140 except:
141 141 continue
142 142
143 143 if (os.path.splitext(file)[-1].lower() != ext.lower()):
144 144 continue
145 145
146 146 validFilelist.append(file)
147 147
148 148 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
149 149
150 150 if len(myfile)!= 0:
151 151 return myfile[0]
152 152 else:
153 153 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
154 154 print 'the filename %s does not exist'%filename
155 155 print '...going to the last file: '
156 156
157 157 if validFilelist:
158 158 validFilelist = sorted( validFilelist, key=str.lower )
159 159 return validFilelist[-1]
160 160
161 161 return None
162 162
163 163
164 164 def getlastFileFromPath(path, ext):
165 165 """
166 166 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
167 167 al final de la depuracion devuelve el ultimo file de la lista que quedo.
168 168
169 169 Input:
170 170 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
171 171 ext : extension de los files contenidos en una carpeta
172 172
173 173 Return:
174 174 El ultimo file de una determinada carpeta, no se considera el path.
175 175 """
176 176 validFilelist = []
177 177 fileList = os.listdir(path)
178 178
179 179 # 0 1234 567 89A BCDE
180 180 # H YYYY DDD SSS .ext
181 181
182 182 for file in fileList:
183 183 try:
184 184 year = int(file[1:5])
185 185 doy = int(file[5:8])
186 186
187 187
188 188 except:
189 189 continue
190 190
191 191 if (os.path.splitext(file)[-1].lower() != ext.lower()):
192 192 continue
193 193
194 194 validFilelist.append(file)
195 195
196 196 if validFilelist:
197 197 validFilelist = sorted( validFilelist, key=str.lower )
198 198 return validFilelist[-1]
199 199
200 200 return None
201 201
202 202 def checkForRealPath(path, foldercounter, year, doy, set, ext):
203 203 """
204 204 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
205 205 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
206 206 el path exacto de un determinado file.
207 207
208 208 Example :
209 209 nombre correcto del file es .../.../D2009307/P2009307367.ext
210 210
211 211 Entonces la funcion prueba con las siguientes combinaciones
212 212 .../.../y2009307367.ext
213 213 .../.../Y2009307367.ext
214 214 .../.../x2009307/y2009307367.ext
215 215 .../.../x2009307/Y2009307367.ext
216 216 .../.../X2009307/y2009307367.ext
217 217 .../.../X2009307/Y2009307367.ext
218 218 siendo para este caso, la ultima combinacion de letras, identica al file buscado
219 219
220 220 Return:
221 221 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
222 222 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
223 223 para el filename
224 224 """
225 225 fullfilename = None
226 226 find_flag = False
227 227 filename = None
228 228
229 229 prefixDirList = [None,'d','D']
230 230 if ext.lower() == ".r": #voltage
231 231 prefixFileList = ['d','D']
232 232 elif ext.lower() == ".pdata": #spectra
233 233 prefixFileList = ['p','P']
234 234 else:
235 235 return None, filename
236 236
237 237 #barrido por las combinaciones posibles
238 238 for prefixDir in prefixDirList:
239 239 thispath = path
240 240 if prefixDir != None:
241 241 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
242 242 if foldercounter == 0:
243 243 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
244 244 else:
245 245 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
246 246 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
247 247 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
248 248 fullfilename = os.path.join( thispath, filename ) #formo el path completo
249 249
250 250 if os.path.exists( fullfilename ): #verifico que exista
251 251 find_flag = True
252 252 break
253 253 if find_flag:
254 254 break
255 255
256 256 if not(find_flag):
257 257 return None, filename
258 258
259 259 return fullfilename, filename
260 260
261 261 def isDoyFolder(folder):
262 262 try:
263 263 year = int(folder[1:5])
264 264 except:
265 265 return 0
266 266
267 267 try:
268 268 doy = int(folder[5:8])
269 269 except:
270 270 return 0
271 271
272 272 return 1
273 273
274 274 class JRODataIO:
275 275
276 276 c = 3E8
277 277
278 278 isConfig = False
279 279
280 280 basicHeaderObj = BasicHeader(LOCALTIME)
281 281
282 282 systemHeaderObj = SystemHeader()
283 283
284 284 radarControllerHeaderObj = RadarControllerHeader()
285 285
286 286 processingHeaderObj = ProcessingHeader()
287 287
288 288 online = 0
289 289
290 290 dtype = None
291 291
292 292 pathList = []
293 293
294 294 filenameList = []
295 295
296 296 filename = None
297 297
298 298 ext = None
299 299
300 300 flagIsNewFile = 1
301 301
302 302 flagTimeBlock = 0
303 303
304 304 flagIsNewBlock = 0
305 305
306 306 fp = None
307 307
308 308 firstHeaderSize = 0
309 309
310 310 basicHeaderSize = 24
311 311
312 312 versionFile = 1103
313 313
314 314 fileSize = None
315 315
316 316 ippSeconds = None
317 317
318 318 fileSizeByHeader = None
319 319
320 320 fileIndex = None
321 321
322 322 profileIndex = None
323 323
324 324 blockIndex = None
325 325
326 326 nTotalBlocks = None
327 327
328 328 maxTimeStep = 30
329 329
330 330 lastUTTime = None
331 331
332 332 datablock = None
333 333
334 334 dataOut = None
335 335
336 336 blocksize = None
337 337
338 338 def __init__(self):
339 339
340 340 raise ValueError, "Not implemented"
341 341
342 342 def run(self):
343 343
344 344 raise ValueError, "Not implemented"
345 345
346 346 def getOutput(self):
347 347
348 348 return self.dataOut
349 349
350 350 class JRODataReader(JRODataIO, ProcessingUnit):
351 351
352 352 nReadBlocks = 0
353 353
354 354 delay = 10 #number of seconds waiting a new file
355 355
356 356 nTries = 3 #quantity tries
357 357
358 358 nFiles = 3 #number of files for searching
359 359
360 360 path = None
361 361
362 362 foldercounter = 0
363 363
364 364 flagNoMoreFiles = 0
365 365
366 366 datetimeList = []
367 367
368 368 __isFirstTimeOnline = 1
369 369
370 370 __printInfo = True
371 371
372 372 profileIndex = None
373 373
374 374 def __init__(self):
375 375
376 376 """
377 377
378 378 """
379 379
380 380 raise ValueError, "This method has not been implemented"
381 381
382 382
383 383 def createObjByDefault(self):
384 384 """
385 385
386 386 """
387 387 raise ValueError, "This method has not been implemented"
388 388
389 389 def getBlockDimension(self):
390 390
391 391 raise ValueError, "No implemented"
392 392
393 393 def __searchFilesOffLine(self,
394 394 path,
395 395 startDate,
396 396 endDate,
397 397 startTime=datetime.time(0,0,0),
398 398 endTime=datetime.time(23,59,59),
399 399 set=None,
400 400 expLabel='',
401 401 ext='.r',
402 402 walk=True):
403 403
404 404 pathList = []
405 405
406 406 if not walk:
407 407 #pathList.append(path)
408 408 multi_path = path.split(',')
409 409 for single_path in multi_path:
410 410 pathList.append(single_path)
411 411
412 412 else:
413 413 #dirList = []
414 414 multi_path = path.split(',')
415 415 for single_path in multi_path:
416 416 dirList = []
417 417 for thisPath in os.listdir(single_path):
418 418 if not os.path.isdir(os.path.join(single_path,thisPath)):
419 419 continue
420 420 if not isDoyFolder(thisPath):
421 421 continue
422 422
423 423 dirList.append(thisPath)
424 424
425 425 if not(dirList):
426 426 return None, None
427 427
428 428 thisDate = startDate
429 429
430 430 while(thisDate <= endDate):
431 431 year = thisDate.timetuple().tm_year
432 432 doy = thisDate.timetuple().tm_yday
433 433
434 434 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
435 435 if len(matchlist) == 0:
436 436 thisDate += datetime.timedelta(1)
437 437 continue
438 438 for match in matchlist:
439 439 pathList.append(os.path.join(single_path,match,expLabel))
440 440
441 441 thisDate += datetime.timedelta(1)
442 442
443 443 if pathList == []:
444 444 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
445 445 return None, None
446 446
447 447 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
448 448
449 449 filenameList = []
450 450 datetimeList = []
451 451 pathDict = {}
452 452 filenameList_to_sort = []
453 453
454 454 for i in range(len(pathList)):
455 455
456 456 thisPath = pathList[i]
457 457
458 458 fileList = glob.glob1(thisPath, "*%s" %ext)
459 459 fileList.sort()
460 460 pathDict.setdefault(fileList[0])
461 461 pathDict[fileList[0]] = i
462 462 filenameList_to_sort.append(fileList[0])
463 463
464 464 filenameList_to_sort.sort()
465 465
466 466 for file in filenameList_to_sort:
467 467 thisPath = pathList[pathDict[file]]
468 468
469 469 fileList = glob.glob1(thisPath, "*%s" %ext)
470 470 fileList.sort()
471 471
472 472 for file in fileList:
473 473
474 474 filename = os.path.join(thisPath,file)
475 475 thisDatetime = isFileinThisTime(filename, startTime, endTime)
476 476
477 477 if not(thisDatetime):
478 478 continue
479 479
480 480 filenameList.append(filename)
481 481 datetimeList.append(thisDatetime)
482 482
483 483 if not(filenameList):
484 484 print "Any file was found for the time range %s - %s" %(startTime, endTime)
485 485 return None, None
486 486
487 487 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
488 488 print
489 489
490 490 for i in range(len(filenameList)):
491 491 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
492 492
493 493 self.filenameList = filenameList
494 494 self.datetimeList = datetimeList
495 495
496 496 return pathList, filenameList
497 497
498 498 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
499 499
500 500 """
501 501 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
502 502 devuelve el archivo encontrado ademas de otros datos.
503 503
504 504 Input:
505 505 path : carpeta donde estan contenidos los files que contiene data
506 506
507 507 expLabel : Nombre del subexperimento (subfolder)
508 508
509 509 ext : extension de los files
510 510
511 511 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
512 512
513 513 Return:
514 514 directory : eL directorio donde esta el file encontrado
515 515 filename : el ultimo file de una determinada carpeta
516 516 year : el anho
517 517 doy : el numero de dia del anho
518 518 set : el set del archivo
519 519
520 520
521 521 """
522 522 dirList = []
523 523
524 524 if not walk:
525 525 fullpath = path
526 526 foldercounter = 0
527 527 else:
528 528 #Filtra solo los directorios
529 529 for thisPath in os.listdir(path):
530 530 if not os.path.isdir(os.path.join(path,thisPath)):
531 531 continue
532 532 if not isDoyFolder(thisPath):
533 533 continue
534 534
535 535 dirList.append(thisPath)
536 536
537 537 if not(dirList):
538 538 return None, None, None, None, None, None
539 539
540 540 dirList = sorted( dirList, key=str.lower )
541 541
542 542 doypath = dirList[-1]
543 543 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
544 544 fullpath = os.path.join(path, doypath, expLabel)
545 545
546 546
547 547 print "%s folder was found: " %(fullpath )
548 548
549 549 if set == None:
550 550 filename = getlastFileFromPath(fullpath, ext)
551 551 else:
552 552 filename = getFileFromSet(fullpath, ext, set)
553 553
554 554 if not(filename):
555 555 return None, None, None, None, None, None
556 556
557 557 print "%s file was found" %(filename)
558 558
559 559 if not(self.__verifyFile(os.path.join(fullpath, filename))):
560 560 return None, None, None, None, None, None
561 561
562 562 year = int( filename[1:5] )
563 563 doy = int( filename[5:8] )
564 564 set = int( filename[8:11] )
565 565
566 566 return fullpath, foldercounter, filename, year, doy, set
567 567
568 568 def __setNextFileOffline(self):
569 569
570 570 idFile = self.fileIndex
571 571
572 572 while (True):
573 573 idFile += 1
574 574 if not(idFile < len(self.filenameList)):
575 575 self.flagNoMoreFiles = 1
576 576 print "No more Files"
577 577 return 0
578 578
579 579 filename = self.filenameList[idFile]
580 580
581 581 if not(self.__verifyFile(filename)):
582 582 continue
583 583
584 584 fileSize = os.path.getsize(filename)
585 585 fp = open(filename,'rb')
586 586 break
587 587
588 588 self.flagIsNewFile = 1
589 589 self.fileIndex = idFile
590 590 self.filename = filename
591 591 self.fileSize = fileSize
592 592 self.fp = fp
593 593
594 594 print "Setting the file: %s"%self.filename
595 595
596 596 return 1
597 597
598 598 def __setNextFileOnline(self):
599 599 """
600 600 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
601 601 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
602 602 siguientes.
603 603
604 604 Affected:
605 605 self.flagIsNewFile
606 606 self.filename
607 607 self.fileSize
608 608 self.fp
609 609 self.set
610 610 self.flagNoMoreFiles
611 611
612 612 Return:
613 613 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
614 614 1 : si el file fue abierto con exito y esta listo a ser leido
615 615
616 616 Excepciones:
617 617 Si un determinado file no puede ser abierto
618 618 """
619 619 nFiles = 0
620 620 fileOk_flag = False
621 621 firstTime_flag = True
622 622
623 623 self.set += 1
624 624
625 625 if self.set > 999:
626 626 self.set = 0
627 627 self.foldercounter += 1
628 628
629 629 #busca el 1er file disponible
630 630 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
631 631 if fullfilename:
632 632 if self.__verifyFile(fullfilename, False):
633 633 fileOk_flag = True
634 634
635 635 #si no encuentra un file entonces espera y vuelve a buscar
636 636 if not(fileOk_flag):
637 637 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
638 638
639 639 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
640 640 tries = self.nTries
641 641 else:
642 642 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
643 643
644 644 for nTries in range( tries ):
645 645 if firstTime_flag:
646 646 print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
647 647 time.sleep( self.delay )
648 648 else:
649 649 print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
650 650
651 651 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
652 652 if fullfilename:
653 653 if self.__verifyFile(fullfilename):
654 654 fileOk_flag = True
655 655 break
656 656
657 657 if fileOk_flag:
658 658 break
659 659
660 660 firstTime_flag = False
661 661
662 662 print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename
663 663 self.set += 1
664 664
665 665 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
666 666 self.set = 0
667 667 self.doy += 1
668 668 self.foldercounter = 0
669 669
670 670 if fileOk_flag:
671 671 self.fileSize = os.path.getsize( fullfilename )
672 672 self.filename = fullfilename
673 673 self.flagIsNewFile = 1
674 674 if self.fp != None: self.fp.close()
675 675 self.fp = open(fullfilename, 'rb')
676 676 self.flagNoMoreFiles = 0
677 677 print 'Setting the file: %s' % fullfilename
678 678 else:
679 679 self.fileSize = 0
680 680 self.filename = None
681 681 self.flagIsNewFile = 0
682 682 self.fp = None
683 683 self.flagNoMoreFiles = 1
684 684 print 'No more Files'
685 685
686 686 return fileOk_flag
687 687
688 688
689 689 def setNextFile(self):
690 690 if self.fp != None:
691 691 self.fp.close()
692 692
693 693 if self.online:
694 694 newFile = self.__setNextFileOnline()
695 695 else:
696 696 newFile = self.__setNextFileOffline()
697 697
698 698 if not(newFile):
699 699 return 0
700 700
701 701 self.__readFirstHeader()
702 702 self.nReadBlocks = 0
703 703 return 1
704 704
705 705 def __waitNewBlock(self):
706 706 """
707 707 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
708 708
709 709 Si el modo de lectura es OffLine siempre retorn 0
710 710 """
711 711 if not self.online:
712 712 return 0
713 713
714 714 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
715 715 return 0
716 716
717 717 currentPointer = self.fp.tell()
718 718
719 719 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
720 720
721 721 for nTries in range( self.nTries ):
722 722
723 723 self.fp.close()
724 724 self.fp = open( self.filename, 'rb' )
725 725 self.fp.seek( currentPointer )
726 726
727 727 self.fileSize = os.path.getsize( self.filename )
728 728 currentSize = self.fileSize - currentPointer
729 729
730 730 if ( currentSize >= neededSize ):
731 731 self.__rdBasicHeader()
732 732 return 1
733 733
734 734 if self.fileSize == self.fileSizeByHeader:
735 735 # self.flagEoF = True
736 736 return 0
737 737
738 738 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
739 739 time.sleep( self.delay )
740 740
741 741
742 742 return 0
743 743
744 744 def waitDataBlock(self,pointer_location):
745 745
746 746 currentPointer = pointer_location
747 747
748 748 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
749 749
750 750 for nTries in range( self.nTries ):
751 751 self.fp.close()
752 752 self.fp = open( self.filename, 'rb' )
753 753 self.fp.seek( currentPointer )
754 754
755 755 self.fileSize = os.path.getsize( self.filename )
756 756 currentSize = self.fileSize - currentPointer
757 757
758 758 if ( currentSize >= neededSize ):
759 759 return 1
760 760
761 761 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
762 762 time.sleep( self.delay )
763 763
764 764 return 0
765 765
766 766
767 767 def __jumpToLastBlock(self):
768 768
769 769 if not(self.__isFirstTimeOnline):
770 770 return
771 771
772 772 csize = self.fileSize - self.fp.tell()
773 773 blocksize = self.processingHeaderObj.blockSize
774 774
775 775 #salta el primer bloque de datos
776 776 if csize > self.processingHeaderObj.blockSize:
777 777 self.fp.seek(self.fp.tell() + blocksize)
778 778 else:
779 779 return
780 780
781 781 csize = self.fileSize - self.fp.tell()
782 782 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
783 783 while True:
784 784
785 785 if self.fp.tell()<self.fileSize:
786 786 self.fp.seek(self.fp.tell() + neededsize)
787 787 else:
788 788 self.fp.seek(self.fp.tell() - neededsize)
789 789 break
790 790
791 791 # csize = self.fileSize - self.fp.tell()
792 792 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
793 793 # factor = int(csize/neededsize)
794 794 # if factor > 0:
795 795 # self.fp.seek(self.fp.tell() + factor*neededsize)
796 796
797 797 self.flagIsNewFile = 0
798 798 self.__isFirstTimeOnline = 0
799 799
800 800
801 801 def __setNewBlock(self):
802 802
803 803 if self.fp == None:
804 804 return 0
805 805
806 806 if self.online:
807 807 self.__jumpToLastBlock()
808 808
809 809 if self.flagIsNewFile:
810 810 return 1
811 811
812 812 self.lastUTTime = self.basicHeaderObj.utc
813 813 currentSize = self.fileSize - self.fp.tell()
814 814 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
815 815
816 816 if (currentSize >= neededSize):
817 817 self.__rdBasicHeader()
818 818 return 1
819 819
820 820 if self.__waitNewBlock():
821 821 return 1
822 822
823 823 if not(self.setNextFile()):
824 824 return 0
825 825
826 826 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
827 827
828 828 self.flagTimeBlock = 0
829 829
830 830 if deltaTime > self.maxTimeStep:
831 831 self.flagTimeBlock = 1
832 832
833 833 return 1
834 834
835 835
836 836 def readNextBlock(self):
837 837 if not(self.__setNewBlock()):
838 838 return 0
839 839
840 840 if not(self.readBlock()):
841 841 return 0
842 842
843 843 return 1
844 844
845 845 def __rdProcessingHeader(self, fp=None):
846 846 if fp == None:
847 847 fp = self.fp
848 848
849 849 self.processingHeaderObj.read(fp)
850 850
851 851 def __rdRadarControllerHeader(self, fp=None):
852 852 if fp == None:
853 853 fp = self.fp
854 854
855 855 self.radarControllerHeaderObj.read(fp)
856 856
857 857 def __rdSystemHeader(self, fp=None):
858 858 if fp == None:
859 859 fp = self.fp
860 860
861 861 self.systemHeaderObj.read(fp)
862 862
863 863 def __rdBasicHeader(self, fp=None):
864 864 if fp == None:
865 865 fp = self.fp
866 866
867 867 self.basicHeaderObj.read(fp)
868 868
869 869
870 870 def __readFirstHeader(self):
871 871 self.__rdBasicHeader()
872 872 self.__rdSystemHeader()
873 873 self.__rdRadarControllerHeader()
874 874 self.__rdProcessingHeader()
875 875
876 876 self.firstHeaderSize = self.basicHeaderObj.size
877 877
878 878 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
879 879 if datatype == 0:
880 880 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
881 881 elif datatype == 1:
882 882 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
883 883 elif datatype == 2:
884 884 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
885 885 elif datatype == 3:
886 886 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
887 887 elif datatype == 4:
888 888 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
889 889 elif datatype == 5:
890 890 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
891 891 else:
892 892 raise ValueError, 'Data type was not defined'
893 893
894 894 self.dtype = datatype_str
895 895 self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
896 896 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
897 897 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
898 898 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
899 899 self.getBlockDimension()
900 900
901 901
902 902 def __verifyFile(self, filename, msgFlag=True):
903 903 msg = None
904 904 try:
905 905 fp = open(filename, 'rb')
906 906 currentPosition = fp.tell()
907 907 except:
908 908 if msgFlag:
909 909 print "The file %s can't be opened" % (filename)
910 910 return False
911 911
912 912 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
913 913
914 914 if neededSize == 0:
915 915 basicHeaderObj = BasicHeader(LOCALTIME)
916 916 systemHeaderObj = SystemHeader()
917 917 radarControllerHeaderObj = RadarControllerHeader()
918 918 processingHeaderObj = ProcessingHeader()
919 919
920 920 try:
921 921 if not( basicHeaderObj.read(fp) ): raise IOError
922 922 if not( systemHeaderObj.read(fp) ): raise IOError
923 923 if not( radarControllerHeaderObj.read(fp) ): raise IOError
924 924 if not( processingHeaderObj.read(fp) ): raise IOError
925 925 data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
926 926
927 927 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
928 928
929 929 except:
930 930 if msgFlag:
931 931 print "\tThe file %s is empty or it hasn't enough data" % filename
932 932
933 933 fp.close()
934 934 return False
935 935 else:
936 936 msg = "\tSkipping the file %s due to it hasn't enough data" %filename
937 937
938 938 fp.close()
939 939 fileSize = os.path.getsize(filename)
940 940 currentSize = fileSize - currentPosition
941 941 if currentSize < neededSize:
942 942 if msgFlag and (msg != None):
943 943 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
944 944 return False
945 945
946 946 return True
947 947
948 948 def setup(self,
949 949 path=None,
950 950 startDate=None,
951 951 endDate=None,
952 952 startTime=datetime.time(0,0,0),
953 953 endTime=datetime.time(23,59,59),
954 954 set=None,
955 955 expLabel = "",
956 956 ext = None,
957 957 online = False,
958 958 delay = 60,
959 959 walk = True):
960 960
961 961 if path == None:
962 962 raise ValueError, "The path is not valid"
963 963
964 964 if ext == None:
965 965 ext = self.ext
966 966
967 967 if online:
968 968 print "Searching files in online mode..."
969 969
970 970 for nTries in range( self.nTries ):
971 971 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
972 972
973 973 if fullpath:
974 974 break
975 975
976 976 print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
977 977 time.sleep( self.delay )
978 978
979 979 if not(fullpath):
980 980 print "There 'isn't valied files in %s" % path
981 981 return None
982 982
983 983 self.year = year
984 984 self.doy = doy
985 985 self.set = set - 1
986 986 self.path = path
987 987 self.foldercounter = foldercounter
988 988 last_set = None
989 989
990 990 else:
991 991 print "Searching files in offline mode ..."
992 992 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
993 993 startTime=startTime, endTime=endTime,
994 994 set=set, expLabel=expLabel, ext=ext,
995 995 walk=walk)
996 996
997 997 if not(pathList):
998 998 print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
999 999 datetime.datetime.combine(startDate,startTime).ctime(),
1000 1000 datetime.datetime.combine(endDate,endTime).ctime())
1001 1001
1002 1002 sys.exit(-1)
1003 1003
1004 1004
1005 1005 self.fileIndex = -1
1006 1006 self.pathList = pathList
1007 1007 self.filenameList = filenameList
1008 1008 file_name = os.path.basename(filenameList[-1])
1009 1009 basename, ext = os.path.splitext(file_name)
1010 1010 last_set = int(basename[-3:])
1011 1011
1012 1012 self.online = online
1013 1013 self.delay = delay
1014 1014 ext = ext.lower()
1015 1015 self.ext = ext
1016 1016
1017 1017 if not(self.setNextFile()):
1018 1018 if (startDate!=None) and (endDate!=None):
1019 1019 print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1020 1020 elif startDate != None:
1021 1021 print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1022 1022 else:
1023 1023 print "No files"
1024 1024
1025 1025 sys.exit(-1)
1026 1026
1027 1027 # self.updateDataHeader()
1028 1028 if last_set != None:
1029 1029 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1030 1030 return self.dataOut
1031 1031
1032 1032 def getBasicHeader(self):
1033 1033
1034 1034 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds
1035 1035
1036 1036 self.dataOut.flagTimeBlock = self.flagTimeBlock
1037 1037
1038 1038 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1039 1039
1040 1040 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1041 1041
1042 1042 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1043 1043
1044 1044 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1045 1045
1046 1046 def getFirstHeader(self):
1047 1047
1048 1048 raise ValueError, "This method has not been implemented"
1049 1049
1050 1050 def getData():
1051 1051
1052 1052 raise ValueError, "This method has not been implemented"
1053 1053
1054 1054 def hasNotDataInBuffer():
1055 1055
1056 1056 raise ValueError, "This method has not been implemented"
1057 1057
1058 1058 def readBlock():
1059 1059
1060 1060 raise ValueError, "This method has not been implemented"
1061 1061
1062 1062 def isEndProcess(self):
1063 1063
1064 1064 return self.flagNoMoreFiles
1065 1065
1066 1066 def printReadBlocks(self):
1067 1067
1068 1068 print "Number of read blocks per file %04d" %self.nReadBlocks
1069 1069
1070 1070 def printTotalBlocks(self):
1071 1071
1072 1072 print "Number of read blocks %04d" %self.nTotalBlocks
1073 1073
1074 1074 def printNumberOfBlock(self):
1075 1075
1076 1076 if self.flagIsNewBlock:
1077 1077 print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
1078 1078 self.dataOut.blocknow = self.basicHeaderObj.dataBlock
1079 1079 def printInfo(self):
1080 1080
1081 1081 if self.__printInfo == False:
1082 1082 return
1083 1083
1084 1084 self.basicHeaderObj.printInfo()
1085 1085 self.systemHeaderObj.printInfo()
1086 1086 self.radarControllerHeaderObj.printInfo()
1087 1087 self.processingHeaderObj.printInfo()
1088 1088
1089 1089 self.__printInfo = False
1090 1090
1091 1091
1092 1092 def run(self, **kwargs):
1093 1093
1094 1094 if not(self.isConfig):
1095 1095
1096 1096 # self.dataOut = dataOut
1097 1097 self.setup(**kwargs)
1098 1098 self.isConfig = True
1099 1099
1100 1100 self.getData()
1101 1101
1102 1102 class JRODataWriter(JRODataIO, Operation):
1103 1103
1104 1104 """
1105 1105 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1106 1106 de los datos siempre se realiza por bloques.
1107 1107 """
1108 1108
1109 1109 blockIndex = 0
1110 1110
1111 1111 path = None
1112 1112
1113 1113 setFile = None
1114 1114
1115 1115 profilesPerBlock = None
1116 1116
1117 1117 blocksPerFile = None
1118 1118
1119 1119 nWriteBlocks = 0
1120 1120
1121 1121 def __init__(self, dataOut=None):
1122 1122 raise ValueError, "Not implemented"
1123 1123
1124 1124
1125 1125 def hasAllDataInBuffer(self):
1126 1126 raise ValueError, "Not implemented"
1127 1127
1128 1128
1129 1129 def setBlockDimension(self):
1130 1130 raise ValueError, "Not implemented"
1131 1131
1132 1132
1133 1133 def writeBlock(self):
1134 1134 raise ValueError, "No implemented"
1135 1135
1136 1136
1137 1137 def putData(self):
1138 1138 raise ValueError, "No implemented"
1139 1139
1140 1140
1141 1141 def setBasicHeader(self):
1142 1142
1143 1143 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1144 1144 self.basicHeaderObj.version = self.versionFile
1145 1145 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1146 1146
1147 1147 utc = numpy.floor(self.dataOut.utctime)
1148 1148 milisecond = (self.dataOut.utctime - utc)* 1000.0
1149 1149
1150 1150 self.basicHeaderObj.utc = utc
1151 1151 self.basicHeaderObj.miliSecond = milisecond
1152 1152 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1153 1153 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1154 1154 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1155 1155
1156 1156 def setFirstHeader(self):
1157 1157 """
1158 1158 Obtiene una copia del First Header
1159 1159
1160 1160 Affected:
1161 1161
1162 1162 self.basicHeaderObj
1163 1163 self.systemHeaderObj
1164 1164 self.radarControllerHeaderObj
1165 1165 self.processingHeaderObj self.
1166 1166
1167 1167 Return:
1168 1168 None
1169 1169 """
1170 1170
1171 1171 raise ValueError, "No implemented"
1172 1172
1173 1173 def __writeFirstHeader(self):
1174 1174 """
1175 1175 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1176 1176
1177 1177 Affected:
1178 1178 __dataType
1179 1179
1180 1180 Return:
1181 1181 None
1182 1182 """
1183 1183
1184 1184 # CALCULAR PARAMETROS
1185 1185
1186 1186 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1187 1187 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1188 1188
1189 1189 self.basicHeaderObj.write(self.fp)
1190 1190 self.systemHeaderObj.write(self.fp)
1191 1191 self.radarControllerHeaderObj.write(self.fp)
1192 1192 self.processingHeaderObj.write(self.fp)
1193 1193
1194 1194 self.dtype = self.dataOut.dtype
1195 1195
1196 1196 def __setNewBlock(self):
1197 1197 """
1198 1198 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1199 1199
1200 1200 Return:
1201 1201 0 : si no pudo escribir nada
1202 1202 1 : Si escribio el Basic el First Header
1203 1203 """
1204 1204 if self.fp == None:
1205 1205 self.setNextFile()
1206 1206
1207 1207 if self.flagIsNewFile:
1208 1208 return 1
1209 1209
1210 1210 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1211 1211 self.basicHeaderObj.write(self.fp)
1212 1212 return 1
1213 1213
1214 1214 if not( self.setNextFile() ):
1215 1215 return 0
1216 1216
1217 1217 return 1
1218 1218
1219 1219
1220 1220 def writeNextBlock(self):
1221 1221 """
1222 1222 Selecciona el bloque siguiente de datos y los escribe en un file
1223 1223
1224 1224 Return:
1225 1225 0 : Si no hizo pudo escribir el bloque de datos
1226 1226 1 : Si no pudo escribir el bloque de datos
1227 1227 """
1228 1228 if not( self.__setNewBlock() ):
1229 1229 return 0
1230 1230
1231 1231 self.writeBlock()
1232 1232
1233 1233 return 1
1234 1234
1235 1235 def setNextFile(self):
1236 1236 """
1237 1237 Determina el siguiente file que sera escrito
1238 1238
1239 1239 Affected:
1240 1240 self.filename
1241 1241 self.subfolder
1242 1242 self.fp
1243 1243 self.setFile
1244 1244 self.flagIsNewFile
1245 1245
1246 1246 Return:
1247 1247 0 : Si el archivo no puede ser escrito
1248 1248 1 : Si el archivo esta listo para ser escrito
1249 1249 """
1250 1250 ext = self.ext
1251 1251 path = self.path
1252 1252
1253 1253 if self.fp != None:
1254 1254 self.fp.close()
1255 1255
1256 1256 timeTuple = time.localtime( self.dataOut.utctime)
1257 1257 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1258 1258
1259 1259 fullpath = os.path.join( path, subfolder )
1260 1260 if not( os.path.exists(fullpath) ):
1261 1261 os.mkdir(fullpath)
1262 1262 self.setFile = -1 #inicializo mi contador de seteo
1263 1263 else:
1264 1264 filesList = os.listdir( fullpath )
1265 1265 if len( filesList ) > 0:
1266 1266 filesList = sorted( filesList, key=str.lower )
1267 1267 filen = filesList[-1]
1268 1268 # el filename debera tener el siguiente formato
1269 1269 # 0 1234 567 89A BCDE (hex)
1270 1270 # x YYYY DDD SSS .ext
1271 1271 if isNumber( filen[8:11] ):
1272 1272 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1273 1273 else:
1274 1274 self.setFile = -1
1275 1275 else:
1276 1276 self.setFile = -1 #inicializo mi contador de seteo
1277 1277
1278 1278 setFile = self.setFile
1279 1279 setFile += 1
1280 1280
1281 1281 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
1282 1282 timeTuple.tm_year,
1283 1283 timeTuple.tm_yday,
1284 1284 setFile,
1285 1285 ext )
1286 1286
1287 1287 filename = os.path.join( path, subfolder, file )
1288 1288
1289 1289 fp = open( filename,'wb' )
1290 1290
1291 1291 self.blockIndex = 0
1292 1292
1293 1293 #guardando atributos
1294 1294 self.filename = filename
1295 1295 self.subfolder = subfolder
1296 1296 self.fp = fp
1297 1297 self.setFile = setFile
1298 1298 self.flagIsNewFile = 1
1299 1299
1300 1300 self.setFirstHeader()
1301 1301
1302 1302 print 'Writing the file: %s'%self.filename
1303 1303
1304 1304 self.__writeFirstHeader()
1305 1305
1306 1306 return 1
1307 1307
1308 1308 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):
1309 1309 """
1310 1310 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1311 1311
1312 1312 Inputs:
1313 1313 path : el path destino en el cual se escribiran los files a crear
1314 1314 format : formato en el cual sera salvado un file
1315 1315 set : el setebo del file
1316 1316
1317 1317 Return:
1318 1318 0 : Si no realizo un buen seteo
1319 1319 1 : Si realizo un buen seteo
1320 1320 """
1321 1321
1322 1322 if ext == None:
1323 1323 ext = self.ext
1324 1324
1325 1325 ext = ext.lower()
1326 1326
1327 1327 self.ext = ext
1328 1328
1329 1329 self.path = path
1330 1330
1331 1331 self.setFile = set - 1
1332 1332
1333 1333 self.blocksPerFile = blocksPerFile
1334 1334
1335 1335 self.profilesPerBlock = profilesPerBlock
1336 1336
1337 1337 self.dataOut = dataOut
1338 1338
1339 1339 if not(self.setNextFile()):
1340 1340 print "There isn't a next file"
1341 1341 return 0
1342 1342
1343 1343 self.setBlockDimension()
1344 1344
1345 1345 return 1
1346 1346
1347 1347 def run(self, dataOut, **kwargs):
1348 1348
1349 1349 if not(self.isConfig):
1350 1350
1351 1351 self.setup(dataOut, **kwargs)
1352 1352 self.isConfig = True
1353 1353
1354 1354 self.putData()
1355 1355
1356 1356 class VoltageReader(JRODataReader):
1357 1357 """
1358 1358 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
1359 1359 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
1360 1360 perfiles*alturas*canales) son almacenados en la variable "buffer".
1361 1361
1362 1362 perfiles * alturas * canales
1363 1363
1364 1364 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
1365 1365 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
1366 1366 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
1367 1367 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
1368 1368
1369 1369 Example:
1370 1370
1371 1371 dpath = "/home/myuser/data"
1372 1372
1373 1373 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
1374 1374
1375 1375 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
1376 1376
1377 1377 readerObj = VoltageReader()
1378 1378
1379 1379 readerObj.setup(dpath, startTime, endTime)
1380 1380
1381 1381 while(True):
1382 1382
1383 1383 #to get one profile
1384 1384 profile = readerObj.getData()
1385 1385
1386 1386 #print the profile
1387 1387 print profile
1388 1388
1389 1389 #If you want to see all datablock
1390 1390 print readerObj.datablock
1391 1391
1392 1392 if readerObj.flagNoMoreFiles:
1393 1393 break
1394 1394
1395 1395 """
1396 1396
1397 1397 ext = ".r"
1398 1398
1399 1399 optchar = "D"
1400 1400 dataOut = None
1401 1401
1402 1402
1403 1403 def __init__(self):
1404 1404 """
1405 1405 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
1406 1406
1407 1407 Input:
1408 1408 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
1409 1409 almacenar un perfil de datos cada vez que se haga un requerimiento
1410 1410 (getData). El perfil sera obtenido a partir del buffer de datos,
1411 1411 si el buffer esta vacio se hara un nuevo proceso de lectura de un
1412 1412 bloque de datos.
1413 1413 Si este parametro no es pasado se creara uno internamente.
1414 1414
1415 1415 Variables afectadas:
1416 1416 self.dataOut
1417 1417
1418 1418 Return:
1419 1419 None
1420 1420 """
1421 1421
1422 1422 self.isConfig = False
1423 1423
1424 1424 self.datablock = None
1425 1425
1426 1426 self.utc = 0
1427 1427
1428 1428 self.ext = ".r"
1429 1429
1430 1430 self.optchar = "D"
1431 1431
1432 1432 self.basicHeaderObj = BasicHeader(LOCALTIME)
1433 1433
1434 1434 self.systemHeaderObj = SystemHeader()
1435 1435
1436 1436 self.radarControllerHeaderObj = RadarControllerHeader()
1437 1437
1438 1438 self.processingHeaderObj = ProcessingHeader()
1439 1439
1440 1440 self.online = 0
1441 1441
1442 1442 self.fp = None
1443 1443
1444 1444 self.idFile = None
1445 1445
1446 1446 self.dtype = None
1447 1447
1448 1448 self.fileSizeByHeader = None
1449 1449
1450 1450 self.filenameList = []
1451 1451
1452 1452 self.filename = None
1453 1453
1454 1454 self.fileSize = None
1455 1455
1456 1456 self.firstHeaderSize = 0
1457 1457
1458 1458 self.basicHeaderSize = 24
1459 1459
1460 1460 self.pathList = []
1461 1461
1462 1462 self.filenameList = []
1463 1463
1464 1464 self.lastUTTime = 0
1465 1465
1466 1466 self.maxTimeStep = 30
1467 1467
1468 1468 self.flagNoMoreFiles = 0
1469 1469
1470 1470 self.set = 0
1471 1471
1472 1472 self.path = None
1473 1473
1474 1474 self.profileIndex = 2**32-1
1475 1475
1476 1476 self.delay = 3 #seconds
1477 1477
1478 1478 self.nTries = 3 #quantity tries
1479 1479
1480 1480 self.nFiles = 3 #number of files for searching
1481 1481
1482 1482 self.nReadBlocks = 0
1483 1483
1484 1484 self.flagIsNewFile = 1
1485 1485
1486 1486 self.__isFirstTimeOnline = 1
1487 1487
1488 1488 self.ippSeconds = 0
1489 1489
1490 1490 self.flagTimeBlock = 0
1491 1491
1492 1492 self.flagIsNewBlock = 0
1493 1493
1494 1494 self.nTotalBlocks = 0
1495 1495
1496 1496 self.blocksize = 0
1497 1497
1498 1498 self.dataOut = self.createObjByDefault()
1499 1499
1500 1500 def createObjByDefault(self):
1501 1501
1502 1502 dataObj = Voltage()
1503 1503
1504 1504 return dataObj
1505 1505
1506 1506 def __hasNotDataInBuffer(self):
1507 1507 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
1508 1508 return 1
1509 1509 return 0
1510 1510
1511 1511
1512 1512 def getBlockDimension(self):
1513 1513 """
1514 1514 Obtiene la cantidad de puntos a leer por cada bloque de datos
1515 1515
1516 1516 Affected:
1517 1517 self.blocksize
1518 1518
1519 1519 Return:
1520 1520 None
1521 1521 """
1522 1522 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
1523 1523 self.blocksize = pts2read
1524 1524
1525 1525
1526 1526 def readBlock(self):
1527 1527 """
1528 1528 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
1529 1529 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
1530 1530 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
1531 1531 es seteado a 0
1532 1532
1533 1533 Inputs:
1534 1534 None
1535 1535
1536 1536 Return:
1537 1537 None
1538 1538
1539 1539 Affected:
1540 1540 self.profileIndex
1541 1541 self.datablock
1542 1542 self.flagIsNewFile
1543 1543 self.flagIsNewBlock
1544 1544 self.nTotalBlocks
1545 1545
1546 1546 Exceptions:
1547 1547 Si un bloque leido no es un bloque valido
1548 1548 """
1549 1549 current_pointer_location = self.fp.tell()
1550 1550 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
1551 1551
1552 1552 try:
1553 1553 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
1554 1554 except:
1555 1555 #print "The read block (%3d) has not enough data" %self.nReadBlocks
1556 1556
1557 1557 if self.waitDataBlock(pointer_location=current_pointer_location):
1558 1558 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
1559 1559 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
1560 1560 # return 0
1561 1561
1562 1562 junk = numpy.transpose(junk, (2,0,1))
1563 1563 self.datablock = junk['real'] + junk['imag']*1j
1564 1564
1565 1565 self.profileIndex = 0
1566 1566
1567 1567 self.flagIsNewFile = 0
1568 1568 self.flagIsNewBlock = 1
1569 1569
1570 1570 self.nTotalBlocks += 1
1571 1571 self.nReadBlocks += 1
1572 1572
1573 1573 return 1
1574 1574
1575 1575 def getFirstHeader(self):
1576 1576
1577 1577 self.dataOut.dtype = self.dtype
1578 1578
1579 1579 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
1580 1580
1581 1581 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
1582 1582
1583 1583 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
1584 1584
1585 1585 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
1586 1586
1587 1587 self.dataOut.ippSeconds = self.ippSeconds
1588 1588
1589 1589 self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt
1590 1590
1591 1591 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
1592 1592
1593 1593 self.dataOut.flagShiftFFT = False
1594 1594
1595 1595 if self.radarControllerHeaderObj.code != None:
1596 1596
1597 1597 self.dataOut.nCode = self.radarControllerHeaderObj.nCode
1598 1598
1599 1599 self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
1600 1600
1601 1601 self.dataOut.code = self.radarControllerHeaderObj.code
1602 1602
1603 1603 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
1604 1604
1605 1605 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
1606 1606
1607 1607 self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
1608 1608
1609 1609 self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip
1610 1610
1611 1611 self.dataOut.flagShiftFFT = False
1612 1612
1613 1613 def getData(self):
1614 1614 """
1615 1615 getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"
1616 1616 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
1617 1617 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
1618 1618
1619 1619 Ademas incrementa el contador del buffer en 1.
1620 1620
1621 1621 Return:
1622 1622 data : retorna un perfil de voltages (alturas * canales) copiados desde el
1623 1623 buffer. Si no hay mas archivos a leer retorna None.
1624 1624
1625 1625 Variables afectadas:
1626 1626 self.dataOut
1627 1627 self.profileIndex
1628 1628
1629 1629 Affected:
1630 1630 self.dataOut
1631 1631 self.profileIndex
1632 1632 self.flagTimeBlock
1633 1633 self.flagIsNewBlock
1634 1634 """
1635 1635
1636 1636 if self.flagNoMoreFiles:
1637 1637 self.dataOut.flagNoData = True
1638 1638 print 'Process finished'
1639 1639 return 0
1640 1640
1641 1641 self.flagTimeBlock = 0
1642 1642 self.flagIsNewBlock = 0
1643 1643
1644 1644 if self.__hasNotDataInBuffer():
1645 1645
1646 1646 if not( self.readNextBlock() ):
1647 1647 return 0
1648 1648
1649 1649 self.getFirstHeader()
1650 1650
1651 1651 if self.datablock == None:
1652 1652 self.dataOut.flagNoData = True
1653 1653 return 0
1654 1654
1655 1655 self.dataOut.data = self.datablock[:,self.profileIndex,:]
1656 1656
1657 1657 self.dataOut.flagNoData = False
1658 1658
1659 1659 self.getBasicHeader()
1660 1660
1661 1661 self.profileIndex += 1
1662 1662
1663 1663 self.dataOut.realtime = self.online
1664 1664
1665 1665 return self.dataOut.data
1666 1666
1667 1667
1668 1668 class VoltageWriter(JRODataWriter):
1669 1669 """
1670 1670 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
1671 1671 de los datos siempre se realiza por bloques.
1672 1672 """
1673 1673
1674 1674 ext = ".r"
1675 1675
1676 1676 optchar = "D"
1677 1677
1678 1678 shapeBuffer = None
1679 1679
1680 1680
1681 1681 def __init__(self):
1682 1682 """
1683 1683 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
1684 1684
1685 1685 Affected:
1686 1686 self.dataOut
1687 1687
1688 1688 Return: None
1689 1689 """
1690 1690
1691 1691 self.nTotalBlocks = 0
1692 1692
1693 1693 self.profileIndex = 0
1694 1694
1695 1695 self.isConfig = False
1696 1696
1697 1697 self.fp = None
1698 1698
1699 1699 self.flagIsNewFile = 1
1700 1700
1701 1701 self.nTotalBlocks = 0
1702 1702
1703 1703 self.flagIsNewBlock = 0
1704 1704
1705 1705 self.setFile = None
1706 1706
1707 1707 self.dtype = None
1708 1708
1709 1709 self.path = None
1710 1710
1711 1711 self.filename = None
1712 1712
1713 1713 self.basicHeaderObj = BasicHeader(LOCALTIME)
1714 1714
1715 1715 self.systemHeaderObj = SystemHeader()
1716 1716
1717 1717 self.radarControllerHeaderObj = RadarControllerHeader()
1718 1718
1719 1719 self.processingHeaderObj = ProcessingHeader()
1720 1720
1721 1721 def hasAllDataInBuffer(self):
1722 1722 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
1723 1723 return 1
1724 1724 return 0
1725 1725
1726 1726
1727 1727 def setBlockDimension(self):
1728 1728 """
1729 1729 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
1730 1730
1731 1731 Affected:
1732 1732 self.shape_spc_Buffer
1733 1733 self.shape_cspc_Buffer
1734 1734 self.shape_dc_Buffer
1735 1735
1736 1736 Return: None
1737 1737 """
1738 1738 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
1739 1739 self.processingHeaderObj.nHeights,
1740 1740 self.systemHeaderObj.nChannels)
1741 1741
1742 1742 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
1743 1743 self.processingHeaderObj.profilesPerBlock,
1744 1744 self.processingHeaderObj.nHeights),
1745 1745 dtype=numpy.dtype('complex64'))
1746 1746
1747 1747
1748 1748 def writeBlock(self):
1749 1749 """
1750 1750 Escribe el buffer en el file designado
1751 1751
1752 1752 Affected:
1753 1753 self.profileIndex
1754 1754 self.flagIsNewFile
1755 1755 self.flagIsNewBlock
1756 1756 self.nTotalBlocks
1757 1757 self.blockIndex
1758 1758
1759 1759 Return: None
1760 1760 """
1761 1761 data = numpy.zeros( self.shapeBuffer, self.dtype )
1762 1762
1763 1763 junk = numpy.transpose(self.datablock, (1,2,0))
1764 1764
1765 1765 data['real'] = junk.real
1766 1766 data['imag'] = junk.imag
1767 1767
1768 1768 data = data.reshape( (-1) )
1769 1769
1770 1770 data.tofile( self.fp )
1771 1771
1772 1772 self.datablock.fill(0)
1773 1773
1774 1774 self.profileIndex = 0
1775 1775 self.flagIsNewFile = 0
1776 1776 self.flagIsNewBlock = 1
1777 1777
1778 1778 self.blockIndex += 1
1779 1779 self.nTotalBlocks += 1
1780 1780
1781 1781 def putData(self):
1782 1782 """
1783 1783 Setea un bloque de datos y luego los escribe en un file
1784 1784
1785 1785 Affected:
1786 1786 self.flagIsNewBlock
1787 1787 self.profileIndex
1788 1788
1789 1789 Return:
1790 1790 0 : Si no hay data o no hay mas files que puedan escribirse
1791 1791 1 : Si se escribio la data de un bloque en un file
1792 1792 """
1793 1793 if self.dataOut.flagNoData:
1794 1794 return 0
1795 1795
1796 1796 self.flagIsNewBlock = 0
1797 1797
1798 1798 if self.dataOut.flagTimeBlock:
1799 1799
1800 1800 self.datablock.fill(0)
1801 1801 self.profileIndex = 0
1802 1802 self.setNextFile()
1803 1803
1804 1804 if self.profileIndex == 0:
1805 1805 self.setBasicHeader()
1806 1806
1807 1807 self.datablock[:,self.profileIndex,:] = self.dataOut.data
1808 1808
1809 1809 self.profileIndex += 1
1810 1810
1811 1811 if self.hasAllDataInBuffer():
1812 1812 #if self.flagIsNewFile:
1813 1813 self.writeNextBlock()
1814 1814 # self.setFirstHeader()
1815 1815
1816 1816 return 1
1817 1817
1818 1818 def __getProcessFlags(self):
1819 1819
1820 1820 processFlags = 0
1821 1821
1822 1822 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
1823 1823 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
1824 1824 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
1825 1825 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
1826 1826 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
1827 1827 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
1828 1828
1829 1829 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
1830 1830
1831 1831
1832 1832
1833 1833 datatypeValueList = [PROCFLAG.DATATYPE_CHAR,
1834 1834 PROCFLAG.DATATYPE_SHORT,
1835 1835 PROCFLAG.DATATYPE_LONG,
1836 1836 PROCFLAG.DATATYPE_INT64,
1837 1837 PROCFLAG.DATATYPE_FLOAT,
1838 1838 PROCFLAG.DATATYPE_DOUBLE]
1839 1839
1840 1840
1841 1841 for index in range(len(dtypeList)):
1842 1842 if self.dataOut.dtype == dtypeList[index]:
1843 1843 dtypeValue = datatypeValueList[index]
1844 1844 break
1845 1845
1846 1846 processFlags += dtypeValue
1847 1847
1848 1848 if self.dataOut.flagDecodeData:
1849 1849 processFlags += PROCFLAG.DECODE_DATA
1850 1850
1851 1851 if self.dataOut.flagDeflipData:
1852 1852 processFlags += PROCFLAG.DEFLIP_DATA
1853 1853
1854 1854 if self.dataOut.code != None:
1855 1855 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1856 1856
1857 1857 if self.dataOut.nCohInt > 1:
1858 1858 processFlags += PROCFLAG.COHERENT_INTEGRATION
1859 1859
1860 1860 return processFlags
1861 1861
1862 1862
1863 1863 def __getBlockSize(self):
1864 1864 '''
1865 1865 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
1866 1866 '''
1867 1867
1868 1868 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
1869 1869 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
1870 1870 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
1871 1871 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
1872 1872 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
1873 1873 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
1874 1874
1875 1875 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
1876 1876 datatypeValueList = [1,2,4,8,4,8]
1877 1877 for index in range(len(dtypeList)):
1878 1878 if self.dataOut.dtype == dtypeList[index]:
1879 1879 datatypeValue = datatypeValueList[index]
1880 1880 break
1881 1881
1882 1882 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * datatypeValue * 2)
1883 1883
1884 1884 return blocksize
1885 1885
1886 1886 def setFirstHeader(self):
1887 1887
1888 1888 """
1889 1889 Obtiene una copia del First Header
1890 1890
1891 1891 Affected:
1892 1892 self.systemHeaderObj
1893 1893 self.radarControllerHeaderObj
1894 1894 self.dtype
1895 1895
1896 1896 Return:
1897 1897 None
1898 1898 """
1899 1899
1900 1900 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
1901 1901 self.systemHeaderObj.nChannels = self.dataOut.nChannels
1902 1902 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
1903 1903
1904 1904 self.setBasicHeader()
1905 1905
1906 1906 processingHeaderSize = 40 # bytes
1907 1907 self.processingHeaderObj.dtype = 0 # Voltage
1908 1908 self.processingHeaderObj.blockSize = self.__getBlockSize()
1909 1909 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
1910 1910 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
1911 1911 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
1912 1912 self.processingHeaderObj.processFlags = self.__getProcessFlags()
1913 1913 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
1914 1914 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
1915 1915 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
1916 1916
1917 1917 # if self.dataOut.code != None:
1918 1918 # self.processingHeaderObj.code = self.dataOut.code
1919 1919 # self.processingHeaderObj.nCode = self.dataOut.nCode
1920 1920 # self.processingHeaderObj.nBaud = self.dataOut.nBaud
1921 1921 # codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)
1922 1922 # processingHeaderSize += codesize
1923 1923
1924 1924 if self.processingHeaderObj.nWindows != 0:
1925 1925 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
1926 1926 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
1927 1927 self.processingHeaderObj.nHeights = self.dataOut.nHeights
1928 1928 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
1929 1929 processingHeaderSize += 12
1930 1930
1931 1931 self.processingHeaderObj.size = processingHeaderSize
1932 1932
1933 1933 class SpectraReader(JRODataReader):
1934 1934 """
1935 1935 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
1936 1936 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
1937 1937 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
1938 1938
1939 1939 paresCanalesIguales * alturas * perfiles (Self Spectra)
1940 1940 paresCanalesDiferentes * alturas * perfiles (Cross Spectra)
1941 1941 canales * alturas (DC Channels)
1942 1942
1943 1943 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
1944 1944 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
1945 1945 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
1946 1946 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
1947 1947
1948 1948 Example:
1949 1949 dpath = "/home/myuser/data"
1950 1950
1951 1951 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
1952 1952
1953 1953 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
1954 1954
1955 1955 readerObj = SpectraReader()
1956 1956
1957 1957 readerObj.setup(dpath, startTime, endTime)
1958 1958
1959 1959 while(True):
1960 1960
1961 1961 readerObj.getData()
1962 1962
1963 1963 print readerObj.data_spc
1964 1964
1965 1965 print readerObj.data_cspc
1966 1966
1967 1967 print readerObj.data_dc
1968 1968
1969 1969 if readerObj.flagNoMoreFiles:
1970 1970 break
1971 1971
1972 1972 """
1973 1973
1974 1974 pts2read_SelfSpectra = 0
1975 1975
1976 1976 pts2read_CrossSpectra = 0
1977 1977
1978 1978 pts2read_DCchannels = 0
1979 1979
1980 1980 ext = ".pdata"
1981 1981
1982 1982 optchar = "P"
1983 1983
1984 1984 dataOut = None
1985 1985
1986 1986 nRdChannels = None
1987 1987
1988 1988 nRdPairs = None
1989 1989
1990 1990 rdPairList = []
1991 1991
1992 1992 def __init__(self):
1993 1993 """
1994 1994 Inicializador de la clase SpectraReader para la lectura de datos de espectros.
1995 1995
1996 1996 Inputs:
1997 1997 dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para
1998 1998 almacenar un perfil de datos cada vez que se haga un requerimiento
1999 1999 (getData). El perfil sera obtenido a partir del buffer de datos,
2000 2000 si el buffer esta vacio se hara un nuevo proceso de lectura de un
2001 2001 bloque de datos.
2002 2002 Si este parametro no es pasado se creara uno internamente.
2003 2003
2004 2004 Affected:
2005 2005 self.dataOut
2006 2006
2007 2007 Return : None
2008 2008 """
2009 2009
2010 2010 self.isConfig = False
2011 2011
2012 2012 self.pts2read_SelfSpectra = 0
2013 2013
2014 2014 self.pts2read_CrossSpectra = 0
2015 2015
2016 2016 self.pts2read_DCchannels = 0
2017 2017
2018 2018 self.datablock = None
2019 2019
2020 2020 self.utc = None
2021 2021
2022 2022 self.ext = ".pdata"
2023 2023
2024 2024 self.optchar = "P"
2025 2025
2026 2026 self.basicHeaderObj = BasicHeader(LOCALTIME)
2027 2027
2028 2028 self.systemHeaderObj = SystemHeader()
2029 2029
2030 2030 self.radarControllerHeaderObj = RadarControllerHeader()
2031 2031
2032 2032 self.processingHeaderObj = ProcessingHeader()
2033 2033
2034 2034 self.online = 0
2035 2035
2036 2036 self.fp = None
2037 2037
2038 2038 self.idFile = None
2039 2039
2040 2040 self.dtype = None
2041 2041
2042 2042 self.fileSizeByHeader = None
2043 2043
2044 2044 self.filenameList = []
2045 2045
2046 2046 self.filename = None
2047 2047
2048 2048 self.fileSize = None
2049 2049
2050 2050 self.firstHeaderSize = 0
2051 2051
2052 2052 self.basicHeaderSize = 24
2053 2053
2054 2054 self.pathList = []
2055 2055
2056 2056 self.lastUTTime = 0
2057 2057
2058 2058 self.maxTimeStep = 30
2059 2059
2060 2060 self.flagNoMoreFiles = 0
2061 2061
2062 2062 self.set = 0
2063 2063
2064 2064 self.path = None
2065 2065
2066 2066 self.delay = 60 #seconds
2067 2067
2068 2068 self.nTries = 3 #quantity tries
2069 2069
2070 2070 self.nFiles = 3 #number of files for searching
2071 2071
2072 2072 self.nReadBlocks = 0
2073 2073
2074 2074 self.flagIsNewFile = 1
2075 2075
2076 2076 self.__isFirstTimeOnline = 1
2077 2077
2078 2078 self.ippSeconds = 0
2079 2079
2080 2080 self.flagTimeBlock = 0
2081 2081
2082 2082 self.flagIsNewBlock = 0
2083 2083
2084 2084 self.nTotalBlocks = 0
2085 2085
2086 2086 self.blocksize = 0
2087 2087
2088 2088 self.dataOut = self.createObjByDefault()
2089 2089
2090 2090 self.profileIndex = 1 #Always
2091 2091
2092 2092
2093 2093 def createObjByDefault(self):
2094 2094
2095 2095 dataObj = Spectra()
2096 2096
2097 2097 return dataObj
2098 2098
2099 2099 def __hasNotDataInBuffer(self):
2100 2100 return 1
2101 2101
2102 2102
2103 2103 def getBlockDimension(self):
2104 2104 """
2105 2105 Obtiene la cantidad de puntos a leer por cada bloque de datos
2106 2106
2107 2107 Affected:
2108 2108 self.nRdChannels
2109 2109 self.nRdPairs
2110 2110 self.pts2read_SelfSpectra
2111 2111 self.pts2read_CrossSpectra
2112 2112 self.pts2read_DCchannels
2113 2113 self.blocksize
2114 2114 self.dataOut.nChannels
2115 2115 self.dataOut.nPairs
2116 2116
2117 2117 Return:
2118 2118 None
2119 2119 """
2120 2120 self.nRdChannels = 0
2121 2121 self.nRdPairs = 0
2122 2122 self.rdPairList = []
2123 2123
2124 2124 for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
2125 2125 if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
2126 2126 self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
2127 2127 else:
2128 2128 self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
2129 2129 self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
2130 2130
2131 2131 pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
2132 2132
2133 2133 self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
2134 2134 self.blocksize = self.pts2read_SelfSpectra
2135 2135
2136 2136 if self.processingHeaderObj.flag_cspc:
2137 2137 self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
2138 2138 self.blocksize += self.pts2read_CrossSpectra
2139 2139
2140 2140 if self.processingHeaderObj.flag_dc:
2141 2141 self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
2142 2142 self.blocksize += self.pts2read_DCchannels
2143 2143
2144 2144 # self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
2145 2145
2146 2146
2147 2147 def readBlock(self):
2148 2148 """
2149 2149 Lee el bloque de datos desde la posicion actual del puntero del archivo
2150 2150 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
2151 2151 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
2152 2152 es seteado a 0
2153 2153
2154 2154 Return: None
2155 2155
2156 2156 Variables afectadas:
2157 2157
2158 2158 self.flagIsNewFile
2159 2159 self.flagIsNewBlock
2160 2160 self.nTotalBlocks
2161 2161 self.data_spc
2162 2162 self.data_cspc
2163 2163 self.data_dc
2164 2164
2165 2165 Exceptions:
2166 2166 Si un bloque leido no es un bloque valido
2167 2167 """
2168 2168 blockOk_flag = False
2169 2169 fpointer = self.fp.tell()
2170 2170
2171 2171 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
2172 2172 spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
2173 2173
2174 2174 if self.processingHeaderObj.flag_cspc:
2175 2175 cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
2176 2176 cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
2177 2177
2178 2178 if self.processingHeaderObj.flag_dc:
2179 2179 dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
2180 2180 dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
2181 2181
2182 2182
2183 2183 if not(self.processingHeaderObj.shif_fft):
2184 2184 #desplaza a la derecha en el eje 2 determinadas posiciones
2185 2185 shift = int(self.processingHeaderObj.profilesPerBlock/2)
2186 2186 spc = numpy.roll( spc, shift , axis=2 )
2187 2187
2188 2188 if self.processingHeaderObj.flag_cspc:
2189 2189 #desplaza a la derecha en el eje 2 determinadas posiciones
2190 2190 cspc = numpy.roll( cspc, shift, axis=2 )
2191 2191
2192 2192 # self.processingHeaderObj.shif_fft = True
2193 2193
2194 2194 spc = numpy.transpose( spc, (0,2,1) )
2195 2195 self.data_spc = spc
2196 2196
2197 2197 if self.processingHeaderObj.flag_cspc:
2198 2198 cspc = numpy.transpose( cspc, (0,2,1) )
2199 2199 self.data_cspc = cspc['real'] + cspc['imag']*1j
2200 2200 else:
2201 2201 self.data_cspc = None
2202 2202
2203 2203 if self.processingHeaderObj.flag_dc:
2204 2204 self.data_dc = dc['real'] + dc['imag']*1j
2205 2205 else:
2206 2206 self.data_dc = None
2207 2207
2208 2208 self.flagIsNewFile = 0
2209 2209 self.flagIsNewBlock = 1
2210 2210
2211 2211 self.nTotalBlocks += 1
2212 2212 self.nReadBlocks += 1
2213 2213
2214 2214 return 1
2215 2215
2216 2216 def getFirstHeader(self):
2217 2217
2218 2218 self.dataOut.dtype = self.dtype
2219 2219
2220 2220 self.dataOut.nPairs = self.nRdPairs
2221 2221
2222 2222 self.dataOut.pairsList = self.rdPairList
2223 2223
2224 2224 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
2225 2225
2226 2226 self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
2227 2227
2228 2228 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
2229 2229
2230 2230 self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
2231 2231
2232 2232 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
2233 2233
2234 2234 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
2235 2235
2236 2236 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
2237 2237
2238 2238 self.dataOut.ippSeconds = self.ippSeconds
2239 2239
2240 2240 self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints
2241 2241
2242 2242 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
2243 2243
2244 2244 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
2245 2245
2246 2246 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
2247 2247
2248 2248 self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
2249 2249
2250 2250 self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip
2251 2251
2252 if self.processingHeaderObj.code != None:
2252 if self.radarControllerHeaderObj.code != None:
2253 2253
2254 self.dataOut.nCode = self.processingHeaderObj.nCode
2254 self.dataOut.nCode = self.radarControllerHeaderObj.nCode
2255 2255
2256 self.dataOut.nBaud = self.processingHeaderObj.nBaud
2256 self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
2257 2257
2258 self.dataOut.code = self.processingHeaderObj.code
2258 self.dataOut.code = self.radarControllerHeaderObj.code
2259 2259
2260 2260 self.dataOut.flagDecodeData = True
2261 2261
2262 2262 def getData(self):
2263 2263 """
2264 2264 Copia el buffer de lectura a la clase "Spectra",
2265 2265 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
2266 2266 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
2267 2267
2268 2268 Return:
2269 2269 0 : Si no hay mas archivos disponibles
2270 2270 1 : Si hizo una buena copia del buffer
2271 2271
2272 2272 Affected:
2273 2273 self.dataOut
2274 2274
2275 2275 self.flagTimeBlock
2276 2276 self.flagIsNewBlock
2277 2277 """
2278 2278
2279 2279 if self.flagNoMoreFiles:
2280 2280 self.dataOut.flagNoData = True
2281 2281 print 'Process finished'
2282 2282 return 0
2283 2283
2284 2284 self.flagTimeBlock = 0
2285 2285 self.flagIsNewBlock = 0
2286 2286
2287 2287 if self.__hasNotDataInBuffer():
2288 2288
2289 2289 if not( self.readNextBlock() ):
2290 2290 self.dataOut.flagNoData = True
2291 2291 return 0
2292 2292
2293 2293 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
2294 2294
2295 2295 if self.data_dc == None:
2296 2296 self.dataOut.flagNoData = True
2297 2297 return 0
2298 2298
2299 2299 self.getBasicHeader()
2300 2300
2301 2301 self.getFirstHeader()
2302 2302
2303 2303 self.dataOut.data_spc = self.data_spc
2304 2304
2305 2305 self.dataOut.data_cspc = self.data_cspc
2306 2306
2307 2307 self.dataOut.data_dc = self.data_dc
2308 2308
2309 2309 self.dataOut.flagNoData = False
2310 2310
2311 2311 self.dataOut.realtime = self.online
2312 2312
2313 2313 return self.dataOut.data_spc
2314 2314
2315 2315
2316 2316 class SpectraWriter(JRODataWriter):
2317 2317
2318 2318 """
2319 2319 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
2320 2320 de los datos siempre se realiza por bloques.
2321 2321 """
2322 2322
2323 2323 ext = ".pdata"
2324 2324
2325 2325 optchar = "P"
2326 2326
2327 2327 shape_spc_Buffer = None
2328 2328
2329 2329 shape_cspc_Buffer = None
2330 2330
2331 2331 shape_dc_Buffer = None
2332 2332
2333 2333 data_spc = None
2334 2334
2335 2335 data_cspc = None
2336 2336
2337 2337 data_dc = None
2338 2338
2339 2339 # dataOut = None
2340 2340
2341 2341 def __init__(self):
2342 2342 """
2343 2343 Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
2344 2344
2345 2345 Affected:
2346 2346 self.dataOut
2347 2347 self.basicHeaderObj
2348 2348 self.systemHeaderObj
2349 2349 self.radarControllerHeaderObj
2350 2350 self.processingHeaderObj
2351 2351
2352 2352 Return: None
2353 2353 """
2354 2354
2355 2355 self.isConfig = False
2356 2356
2357 2357 self.nTotalBlocks = 0
2358 2358
2359 2359 self.data_spc = None
2360 2360
2361 2361 self.data_cspc = None
2362 2362
2363 2363 self.data_dc = None
2364 2364
2365 2365 self.fp = None
2366 2366
2367 2367 self.flagIsNewFile = 1
2368 2368
2369 2369 self.nTotalBlocks = 0
2370 2370
2371 2371 self.flagIsNewBlock = 0
2372 2372
2373 2373 self.setFile = None
2374 2374
2375 2375 self.dtype = None
2376 2376
2377 2377 self.path = None
2378 2378
2379 2379 self.noMoreFiles = 0
2380 2380
2381 2381 self.filename = None
2382 2382
2383 2383 self.basicHeaderObj = BasicHeader(LOCALTIME)
2384 2384
2385 2385 self.systemHeaderObj = SystemHeader()
2386 2386
2387 2387 self.radarControllerHeaderObj = RadarControllerHeader()
2388 2388
2389 2389 self.processingHeaderObj = ProcessingHeader()
2390 2390
2391 2391
2392 2392 def hasAllDataInBuffer(self):
2393 2393 return 1
2394 2394
2395 2395
2396 2396 def setBlockDimension(self):
2397 2397 """
2398 2398 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
2399 2399
2400 2400 Affected:
2401 2401 self.shape_spc_Buffer
2402 2402 self.shape_cspc_Buffer
2403 2403 self.shape_dc_Buffer
2404 2404
2405 2405 Return: None
2406 2406 """
2407 2407 self.shape_spc_Buffer = (self.dataOut.nChannels,
2408 2408 self.processingHeaderObj.nHeights,
2409 2409 self.processingHeaderObj.profilesPerBlock)
2410 2410
2411 2411 self.shape_cspc_Buffer = (self.dataOut.nPairs,
2412 2412 self.processingHeaderObj.nHeights,
2413 2413 self.processingHeaderObj.profilesPerBlock)
2414 2414
2415 2415 self.shape_dc_Buffer = (self.dataOut.nChannels,
2416 2416 self.processingHeaderObj.nHeights)
2417 2417
2418 2418
2419 2419 def writeBlock(self):
2420 2420 """
2421 2421 Escribe el buffer en el file designado
2422 2422
2423 2423 Affected:
2424 2424 self.data_spc
2425 2425 self.data_cspc
2426 2426 self.data_dc
2427 2427 self.flagIsNewFile
2428 2428 self.flagIsNewBlock
2429 2429 self.nTotalBlocks
2430 2430 self.nWriteBlocks
2431 2431
2432 2432 Return: None
2433 2433 """
2434 2434
2435 2435 spc = numpy.transpose( self.data_spc, (0,2,1) )
2436 2436 if not( self.processingHeaderObj.shif_fft ):
2437 2437 spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
2438 2438 data = spc.reshape((-1))
2439 2439 data = data.astype(self.dtype[0])
2440 2440 data.tofile(self.fp)
2441 2441
2442 2442 if self.data_cspc != None:
2443 2443 data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )
2444 2444 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
2445 2445 if not( self.processingHeaderObj.shif_fft ):
2446 2446 cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
2447 2447 data['real'] = cspc.real
2448 2448 data['imag'] = cspc.imag
2449 2449 data = data.reshape((-1))
2450 2450 data.tofile(self.fp)
2451 2451
2452 2452 if self.data_dc != None:
2453 2453 data = numpy.zeros( self.shape_dc_Buffer, self.dtype )
2454 2454 dc = self.data_dc
2455 2455 data['real'] = dc.real
2456 2456 data['imag'] = dc.imag
2457 2457 data = data.reshape((-1))
2458 2458 data.tofile(self.fp)
2459 2459
2460 2460 self.data_spc.fill(0)
2461 2461
2462 2462 if self.data_dc != None:
2463 2463 self.data_dc.fill(0)
2464 2464
2465 2465 if self.data_cspc != None:
2466 2466 self.data_cspc.fill(0)
2467 2467
2468 2468 self.flagIsNewFile = 0
2469 2469 self.flagIsNewBlock = 1
2470 2470 self.nTotalBlocks += 1
2471 2471 self.nWriteBlocks += 1
2472 2472 self.blockIndex += 1
2473 2473
2474 2474
2475 2475 def putData(self):
2476 2476 """
2477 2477 Setea un bloque de datos y luego los escribe en un file
2478 2478
2479 2479 Affected:
2480 2480 self.data_spc
2481 2481 self.data_cspc
2482 2482 self.data_dc
2483 2483
2484 2484 Return:
2485 2485 0 : Si no hay data o no hay mas files que puedan escribirse
2486 2486 1 : Si se escribio la data de un bloque en un file
2487 2487 """
2488 2488
2489 2489 if self.dataOut.flagNoData:
2490 2490 return 0
2491 2491
2492 2492 self.flagIsNewBlock = 0
2493 2493
2494 2494 if self.dataOut.flagTimeBlock:
2495 2495 self.data_spc.fill(0)
2496 2496 self.data_cspc.fill(0)
2497 2497 self.data_dc.fill(0)
2498 2498 self.setNextFile()
2499 2499
2500 2500 if self.flagIsNewFile == 0:
2501 2501 self.setBasicHeader()
2502 2502
2503 2503 self.data_spc = self.dataOut.data_spc.copy()
2504 2504 if self.dataOut.data_cspc != None:
2505 2505 self.data_cspc = self.dataOut.data_cspc.copy()
2506 2506 self.data_dc = self.dataOut.data_dc.copy()
2507 2507
2508 2508 # #self.processingHeaderObj.dataBlocksPerFile)
2509 2509 if self.hasAllDataInBuffer():
2510 2510 # self.setFirstHeader()
2511 2511 self.writeNextBlock()
2512 2512
2513 2513 return 1
2514 2514
2515 2515
2516 2516 def __getProcessFlags(self):
2517 2517
2518 2518 processFlags = 0
2519 2519
2520 2520 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
2521 2521 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
2522 2522 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
2523 2523 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
2524 2524 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
2525 2525 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
2526 2526
2527 2527 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
2528 2528
2529 2529
2530 2530
2531 2531 datatypeValueList = [PROCFLAG.DATATYPE_CHAR,
2532 2532 PROCFLAG.DATATYPE_SHORT,
2533 2533 PROCFLAG.DATATYPE_LONG,
2534 2534 PROCFLAG.DATATYPE_INT64,
2535 2535 PROCFLAG.DATATYPE_FLOAT,
2536 2536 PROCFLAG.DATATYPE_DOUBLE]
2537 2537
2538 2538
2539 2539 for index in range(len(dtypeList)):
2540 2540 if self.dataOut.dtype == dtypeList[index]:
2541 2541 dtypeValue = datatypeValueList[index]
2542 2542 break
2543 2543
2544 2544 processFlags += dtypeValue
2545 2545
2546 2546 if self.dataOut.flagDecodeData:
2547 2547 processFlags += PROCFLAG.DECODE_DATA
2548 2548
2549 2549 if self.dataOut.flagDeflipData:
2550 2550 processFlags += PROCFLAG.DEFLIP_DATA
2551 2551
2552 2552 if self.dataOut.code != None:
2553 2553 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
2554 2554
2555 2555 if self.dataOut.nIncohInt > 1:
2556 2556 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
2557 2557
2558 2558 if self.dataOut.data_dc != None:
2559 2559 processFlags += PROCFLAG.SAVE_CHANNELS_DC
2560 2560
2561 2561 return processFlags
2562 2562
2563 2563
2564 2564 def __getBlockSize(self):
2565 2565 '''
2566 2566 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
2567 2567 '''
2568 2568
2569 2569 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
2570 2570 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
2571 2571 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
2572 2572 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
2573 2573 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
2574 2574 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
2575 2575
2576 2576 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
2577 2577 datatypeValueList = [1,2,4,8,4,8]
2578 2578 for index in range(len(dtypeList)):
2579 2579 if self.dataOut.dtype == dtypeList[index]:
2580 2580 datatypeValue = datatypeValueList[index]
2581 2581 break
2582 2582
2583 2583
2584 2584 pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
2585 2585
2586 2586 pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
2587 2587 blocksize = (pts2write_SelfSpectra*datatypeValue)
2588 2588
2589 2589 if self.dataOut.data_cspc != None:
2590 2590 pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
2591 2591 blocksize += (pts2write_CrossSpectra*datatypeValue*2)
2592 2592
2593 2593 if self.dataOut.data_dc != None:
2594 2594 pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
2595 2595 blocksize += (pts2write_DCchannels*datatypeValue*2)
2596 2596
2597 2597 blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
2598 2598
2599 2599 return blocksize
2600 2600
2601 2601 def setFirstHeader(self):
2602 2602
2603 2603 """
2604 2604 Obtiene una copia del First Header
2605 2605
2606 2606 Affected:
2607 2607 self.systemHeaderObj
2608 2608 self.radarControllerHeaderObj
2609 2609 self.dtype
2610 2610
2611 2611 Return:
2612 2612 None
2613 2613 """
2614 2614
2615 2615 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
2616 2616 self.systemHeaderObj.nChannels = self.dataOut.nChannels
2617 2617 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
2618 2618
2619 2619 self.setBasicHeader()
2620 2620
2621 2621 processingHeaderSize = 40 # bytes
2622 2622 self.processingHeaderObj.dtype = 1 # Spectra
2623 2623 self.processingHeaderObj.blockSize = self.__getBlockSize()
2624 2624 self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
2625 2625 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
2626 2626 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
2627 2627 self.processingHeaderObj.processFlags = self.__getProcessFlags()
2628 2628 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
2629 2629 self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
2630 2630 self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
2631 2631 self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
2632 2632
2633 2633 if self.processingHeaderObj.totalSpectra > 0:
2634 2634 channelList = []
2635 2635 for channel in range(self.dataOut.nChannels):
2636 2636 channelList.append(channel)
2637 2637 channelList.append(channel)
2638 2638
2639 2639 pairsList = []
2640 2640 if self.dataOut.nPairs > 0:
2641 2641 for pair in self.dataOut.pairsList:
2642 2642 pairsList.append(pair[0])
2643 2643 pairsList.append(pair[1])
2644 2644
2645 2645 spectraComb = channelList + pairsList
2646 2646 spectraComb = numpy.array(spectraComb,dtype="u1")
2647 2647 self.processingHeaderObj.spectraComb = spectraComb
2648 2648 sizeOfSpcComb = len(spectraComb)
2649 2649 processingHeaderSize += sizeOfSpcComb
2650 2650
2651 2651 # The processing header should not have information about code
2652 2652 # if self.dataOut.code != None:
2653 2653 # self.processingHeaderObj.code = self.dataOut.code
2654 2654 # self.processingHeaderObj.nCode = self.dataOut.nCode
2655 2655 # self.processingHeaderObj.nBaud = self.dataOut.nBaud
2656 2656 # nCodeSize = 4 # bytes
2657 2657 # nBaudSize = 4 # bytes
2658 2658 # codeSize = 4 # bytes
2659 2659 # sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)
2660 2660 # processingHeaderSize += sizeOfCode
2661 2661
2662 2662 if self.processingHeaderObj.nWindows != 0:
2663 2663 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
2664 2664 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
2665 2665 self.processingHeaderObj.nHeights = self.dataOut.nHeights
2666 2666 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
2667 2667 sizeOfFirstHeight = 4
2668 2668 sizeOfdeltaHeight = 4
2669 2669 sizeOfnHeights = 4
2670 2670 sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows
2671 2671 processingHeaderSize += sizeOfWindows
2672 2672
2673 2673 self.processingHeaderObj.size = processingHeaderSize
2674 2674
2675 2675 class SpectraHeisWriter(Operation):
2676 2676 # set = None
2677 2677 setFile = None
2678 2678 idblock = None
2679 2679 doypath = None
2680 2680 subfolder = None
2681 2681
2682 2682 def __init__(self):
2683 2683 self.wrObj = FITS()
2684 2684 # self.dataOut = dataOut
2685 2685 self.nTotalBlocks=0
2686 2686 # self.set = None
2687 2687 self.setFile = None
2688 2688 self.idblock = 0
2689 2689 self.wrpath = None
2690 2690 self.doypath = None
2691 2691 self.subfolder = None
2692 2692 self.isConfig = False
2693 2693
2694 2694 def isNumber(str):
2695 2695 """
2696 2696 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
2697 2697
2698 2698 Excepciones:
2699 2699 Si un determinado string no puede ser convertido a numero
2700 2700 Input:
2701 2701 str, string al cual se le analiza para determinar si convertible a un numero o no
2702 2702
2703 2703 Return:
2704 2704 True : si el string es uno numerico
2705 2705 False : no es un string numerico
2706 2706 """
2707 2707 try:
2708 2708 float( str )
2709 2709 return True
2710 2710 except:
2711 2711 return False
2712 2712
2713 2713 def setup(self, dataOut, wrpath):
2714 2714
2715 2715 if not(os.path.exists(wrpath)):
2716 2716 os.mkdir(wrpath)
2717 2717
2718 2718 self.wrpath = wrpath
2719 2719 # self.setFile = 0
2720 2720 self.dataOut = dataOut
2721 2721
2722 2722 def putData(self):
2723 2723 name= time.localtime( self.dataOut.utctime)
2724 2724 ext=".fits"
2725 2725
2726 2726 if self.doypath == None:
2727 2727 self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
2728 2728 self.doypath = os.path.join( self.wrpath, self.subfolder )
2729 2729 os.mkdir(self.doypath)
2730 2730
2731 2731 if self.setFile == None:
2732 2732 # self.set = self.dataOut.set
2733 2733 self.setFile = 0
2734 2734 # if self.set != self.dataOut.set:
2735 2735 ## self.set = self.dataOut.set
2736 2736 # self.setFile = 0
2737 2737
2738 2738 #make the filename
2739 2739 file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
2740 2740
2741 2741 filename = os.path.join(self.wrpath,self.subfolder, file)
2742 2742
2743 2743 idblock = numpy.array([self.idblock],dtype="int64")
2744 2744 header=self.wrObj.cFImage(idblock=idblock,
2745 2745 year=time.gmtime(self.dataOut.utctime).tm_year,
2746 2746 month=time.gmtime(self.dataOut.utctime).tm_mon,
2747 2747 day=time.gmtime(self.dataOut.utctime).tm_mday,
2748 2748 hour=time.gmtime(self.dataOut.utctime).tm_hour,
2749 2749 minute=time.gmtime(self.dataOut.utctime).tm_min,
2750 2750 second=time.gmtime(self.dataOut.utctime).tm_sec)
2751 2751
2752 2752 c=3E8
2753 2753 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
2754 2754 freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
2755 2755
2756 2756 colList = []
2757 2757
2758 2758 colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
2759 2759
2760 2760 colList.append(colFreq)
2761 2761
2762 2762 nchannel=self.dataOut.nChannels
2763 2763
2764 2764 for i in range(nchannel):
2765 2765 col = self.wrObj.writeData(name="PCh"+str(i+1),
2766 2766 format=str(self.dataOut.nFFTPoints)+'E',
2767 2767 data=10*numpy.log10(self.dataOut.data_spc[i,:]))
2768 2768
2769 2769 colList.append(col)
2770 2770
2771 2771 data=self.wrObj.Ctable(colList=colList)
2772 2772
2773 2773 self.wrObj.CFile(header,data)
2774 2774
2775 2775 self.wrObj.wFile(filename)
2776 2776
2777 2777 #update the setFile
2778 2778 self.setFile += 1
2779 2779 self.idblock += 1
2780 2780
2781 2781 return 1
2782 2782
2783 2783 def run(self, dataOut, **kwargs):
2784 2784
2785 2785 if not(self.isConfig):
2786 2786
2787 2787 self.setup(dataOut, **kwargs)
2788 2788 self.isConfig = True
2789 2789
2790 2790 self.putData()
2791 2791
2792 2792
2793 2793
2794 2794 class ParameterConf:
2795 2795 ELEMENTNAME = 'Parameter'
2796 2796 def __init__(self):
2797 2797 self.name = ''
2798 2798 self.value = ''
2799 2799
2800 2800 def readXml(self, parmElement):
2801 2801 self.name = parmElement.get('name')
2802 2802 self.value = parmElement.get('value')
2803 2803
2804 2804 def getElementName(self):
2805 2805 return self.ELEMENTNAME
2806 2806
2807 2807 class Metadata:
2808 2808
2809 2809 def __init__(self, filename):
2810 2810 self.parmConfObjList = []
2811 2811 self.readXml(filename)
2812 2812
2813 2813 def readXml(self, filename):
2814 2814 self.projectElement = None
2815 2815 self.procUnitConfObjDict = {}
2816 2816 self.projectElement = ElementTree().parse(filename)
2817 2817 self.project = self.projectElement.tag
2818 2818
2819 2819 parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
2820 2820
2821 2821 for parmElement in parmElementList:
2822 2822 parmConfObj = ParameterConf()
2823 2823 parmConfObj.readXml(parmElement)
2824 2824 self.parmConfObjList.append(parmConfObj)
2825 2825
2826 2826 class FitsWriter(Operation):
2827 2827
2828 2828 def __init__(self):
2829 2829 self.isConfig = False
2830 2830 self.dataBlocksPerFile = None
2831 2831 self.blockIndex = 0
2832 2832 self.flagIsNewFile = 1
2833 2833 self.fitsObj = None
2834 2834 self.optchar = 'P'
2835 2835 self.ext = '.fits'
2836 2836 self.setFile = 0
2837 2837
2838 2838 def setFitsHeader(self, dataOut, metadatafile):
2839 2839
2840 2840 header_data = pyfits.PrimaryHDU()
2841 2841
2842 2842 metadata4fits = Metadata(metadatafile)
2843 2843 for parameter in metadata4fits.parmConfObjList:
2844 2844 parm_name = parameter.name
2845 2845 parm_value = parameter.value
2846 2846
2847 2847 # if parm_value == 'fromdatadatetime':
2848 2848 # value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
2849 2849 # elif parm_value == 'fromdataheights':
2850 2850 # value = dataOut.nHeights
2851 2851 # elif parm_value == 'fromdatachannel':
2852 2852 # value = dataOut.nChannels
2853 2853 # elif parm_value == 'fromdatasamples':
2854 2854 # value = dataOut.nFFTPoints
2855 2855 # else:
2856 2856 # value = parm_value
2857 2857
2858 2858 header_data.header[parm_name] = parm_value
2859 2859
2860 2860
2861 2861 header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
2862 2862 header_data.header['CHANNELLIST'] = str(dataOut.channelList)
2863 2863 header_data.header['NCHANNELS'] = dataOut.nChannels
2864 2864 #header_data.header['HEIGHTS'] = dataOut.heightList
2865 2865 header_data.header['NHEIGHTS'] = dataOut.nHeights
2866 2866
2867 2867 header_data.header['IPPSECONDS'] = dataOut.ippSeconds
2868 2868 header_data.header['NCOHINT'] = dataOut.nCohInt
2869 2869 header_data.header['NINCOHINT'] = dataOut.nIncohInt
2870 2870 header_data.header['TIMEZONE'] = dataOut.timeZone
2871 2871 header_data.header['NBLOCK'] = self.blockIndex
2872 2872
2873 2873 header_data.writeto(self.filename)
2874 2874
2875 2875 self.addExtension(dataOut.heightList,'HEIGHTLIST')
2876 2876
2877 2877
2878 2878 def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):
2879 2879
2880 2880 self.path = path
2881 2881 self.dataOut = dataOut
2882 2882 self.metadatafile = metadatafile
2883 2883 self.dataBlocksPerFile = dataBlocksPerFile
2884 2884
2885 2885 def open(self):
2886 2886 self.fitsObj = pyfits.open(self.filename, mode='update')
2887 2887
2888 2888
2889 2889 def addExtension(self, data, tagname):
2890 2890 self.open()
2891 2891 extension = pyfits.ImageHDU(data=data, name=tagname)
2892 2892 #extension.header['TAG'] = tagname
2893 2893 self.fitsObj.append(extension)
2894 2894 self.write()
2895 2895
2896 2896 def addData(self, data):
2897 2897 self.open()
2898 2898 extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])
2899 2899 extension.header['UTCTIME'] = self.dataOut.utctime
2900 2900 self.fitsObj.append(extension)
2901 2901 self.blockIndex += 1
2902 2902 self.fitsObj[0].header['NBLOCK'] = self.blockIndex
2903 2903
2904 2904 self.write()
2905 2905
2906 2906 def write(self):
2907 2907
2908 2908 self.fitsObj.flush(verbose=True)
2909 2909 self.fitsObj.close()
2910 2910
2911 2911
2912 2912 def setNextFile(self):
2913 2913
2914 2914 ext = self.ext
2915 2915 path = self.path
2916 2916
2917 2917 timeTuple = time.localtime( self.dataOut.utctime)
2918 2918 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
2919 2919
2920 2920 fullpath = os.path.join( path, subfolder )
2921 2921 if not( os.path.exists(fullpath) ):
2922 2922 os.mkdir(fullpath)
2923 2923 self.setFile = -1 #inicializo mi contador de seteo
2924 2924 else:
2925 2925 filesList = os.listdir( fullpath )
2926 2926 if len( filesList ) > 0:
2927 2927 filesList = sorted( filesList, key=str.lower )
2928 2928 filen = filesList[-1]
2929 2929
2930 2930 if isNumber( filen[8:11] ):
2931 2931 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
2932 2932 else:
2933 2933 self.setFile = -1
2934 2934 else:
2935 2935 self.setFile = -1 #inicializo mi contador de seteo
2936 2936
2937 2937 setFile = self.setFile
2938 2938 setFile += 1
2939 2939
2940 2940 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
2941 2941 timeTuple.tm_year,
2942 2942 timeTuple.tm_yday,
2943 2943 setFile,
2944 2944 ext )
2945 2945
2946 2946 filename = os.path.join( path, subfolder, file )
2947 2947
2948 2948 self.blockIndex = 0
2949 2949 self.filename = filename
2950 2950 self.setFile = setFile
2951 2951 self.flagIsNewFile = 1
2952 2952
2953 2953 print 'Writing the file: %s'%self.filename
2954 2954
2955 2955 self.setFitsHeader(self.dataOut, self.metadatafile)
2956 2956
2957 2957 return 1
2958 2958
2959 2959 def writeBlock(self):
2960 2960 self.addData(self.dataOut.data_spc)
2961 2961 self.flagIsNewFile = 0
2962 2962
2963 2963
2964 2964 def __setNewBlock(self):
2965 2965
2966 2966 if self.flagIsNewFile:
2967 2967 return 1
2968 2968
2969 2969 if self.blockIndex < self.dataBlocksPerFile:
2970 2970 return 1
2971 2971
2972 2972 if not( self.setNextFile() ):
2973 2973 return 0
2974 2974
2975 2975 return 1
2976 2976
2977 2977 def writeNextBlock(self):
2978 2978 if not( self.__setNewBlock() ):
2979 2979 return 0
2980 2980 self.writeBlock()
2981 2981 return 1
2982 2982
2983 2983 def putData(self):
2984 2984 if self.flagIsNewFile:
2985 2985 self.setNextFile()
2986 2986 self.writeNextBlock()
2987 2987
2988 2988 def run(self, dataOut, **kwargs):
2989 2989 if not(self.isConfig):
2990 2990 self.setup(dataOut, **kwargs)
2991 2991 self.isConfig = True
2992 2992 self.putData()
2993 2993
2994 2994
2995 2995 class FitsReader(ProcessingUnit):
2996 2996
2997 2997 # __TIMEZONE = time.timezone
2998 2998
2999 2999 expName = None
3000 3000 datetimestr = None
3001 3001 utc = None
3002 3002 nChannels = None
3003 3003 nSamples = None
3004 3004 dataBlocksPerFile = None
3005 3005 comments = None
3006 3006 lastUTTime = None
3007 3007 header_dict = None
3008 3008 data = None
3009 3009 data_header_dict = None
3010 3010
3011 3011 def __init__(self):
3012 3012 self.isConfig = False
3013 3013 self.ext = '.fits'
3014 3014 self.setFile = 0
3015 3015 self.flagNoMoreFiles = 0
3016 3016 self.flagIsNewFile = 1
3017 3017 self.flagTimeBlock = None
3018 3018 self.fileIndex = None
3019 3019 self.filename = None
3020 3020 self.fileSize = None
3021 3021 self.fitsObj = None
3022 3022 self.timeZone = None
3023 3023 self.nReadBlocks = 0
3024 3024 self.nTotalBlocks = 0
3025 3025 self.dataOut = self.createObjByDefault()
3026 3026 self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()
3027 3027 self.blockIndex = 1
3028 3028
3029 3029 def createObjByDefault(self):
3030 3030
3031 3031 dataObj = Fits()
3032 3032
3033 3033 return dataObj
3034 3034
3035 3035 def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):
3036 3036 try:
3037 3037 fitsObj = pyfits.open(filename,'readonly')
3038 3038 except:
3039 3039 raise IOError, "The file %s can't be opened" %(filename)
3040 3040
3041 3041 header = fitsObj[0].header
3042 3042 struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")
3043 3043 utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS
3044 3044
3045 3045 ltc = utc
3046 3046 if useLocalTime:
3047 3047 ltc -= time.timezone
3048 3048 thisDatetime = datetime.datetime.utcfromtimestamp(ltc)
3049 3049 thisTime = thisDatetime.time()
3050 3050
3051 3051 if not ((startTime <= thisTime) and (endTime > thisTime)):
3052 3052 return None
3053 3053
3054 3054 return thisDatetime
3055 3055
3056 3056 def __setNextFileOnline(self):
3057 3057 raise ValueError, "No implemented"
3058 3058
3059 3059 def __setNextFileOffline(self):
3060 3060 idFile = self.fileIndex
3061 3061
3062 3062 while (True):
3063 3063 idFile += 1
3064 3064 if not(idFile < len(self.filenameList)):
3065 3065 self.flagNoMoreFiles = 1
3066 3066 print "No more Files"
3067 3067 return 0
3068 3068
3069 3069 filename = self.filenameList[idFile]
3070 3070
3071 3071 # if not(self.__verifyFile(filename)):
3072 3072 # continue
3073 3073
3074 3074 fileSize = os.path.getsize(filename)
3075 3075 fitsObj = pyfits.open(filename,'readonly')
3076 3076 break
3077 3077
3078 3078 self.flagIsNewFile = 1
3079 3079 self.fileIndex = idFile
3080 3080 self.filename = filename
3081 3081 self.fileSize = fileSize
3082 3082 self.fitsObj = fitsObj
3083 3083 self.blockIndex = 0
3084 3084 print "Setting the file: %s"%self.filename
3085 3085
3086 3086 return 1
3087 3087
3088 3088 def readHeader(self):
3089 3089 headerObj = self.fitsObj[0]
3090 3090
3091 3091 self.header_dict = headerObj.header
3092 3092 if 'EXPNAME' in headerObj.header.keys():
3093 3093 self.expName = headerObj.header['EXPNAME']
3094 3094
3095 3095 if 'DATATYPE' in headerObj.header.keys():
3096 3096 self.dataType = headerObj.header['DATATYPE']
3097 3097
3098 3098 self.datetimestr = headerObj.header['DATETIME']
3099 3099 channelList = headerObj.header['CHANNELLIST']
3100 3100 channelList = channelList.split('[')
3101 3101 channelList = channelList[1].split(']')
3102 3102 channelList = channelList[0].split(',')
3103 3103 channelList = [int(ch) for ch in channelList]
3104 3104 self.channelList = channelList
3105 3105 self.nChannels = headerObj.header['NCHANNELS']
3106 3106 self.nHeights = headerObj.header['NHEIGHTS']
3107 3107 self.ippSeconds = headerObj.header['IPPSECONDS']
3108 3108 self.nCohInt = headerObj.header['NCOHINT']
3109 3109 self.nIncohInt = headerObj.header['NINCOHINT']
3110 3110 self.dataBlocksPerFile = headerObj.header['NBLOCK']
3111 3111 self.timeZone = headerObj.header['TIMEZONE']
3112 3112
3113 3113 self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
3114 3114
3115 3115 if 'COMMENT' in headerObj.header.keys():
3116 3116 self.comments = headerObj.header['COMMENT']
3117 3117
3118 3118 self.readHeightList()
3119 3119
3120 3120 def readHeightList(self):
3121 3121 self.blockIndex = self.blockIndex + 1
3122 3122 obj = self.fitsObj[self.blockIndex]
3123 3123 self.heightList = obj.data
3124 3124 self.blockIndex = self.blockIndex + 1
3125 3125
3126 3126 def readExtension(self):
3127 3127 obj = self.fitsObj[self.blockIndex]
3128 3128 self.heightList = obj.data
3129 3129 self.blockIndex = self.blockIndex + 1
3130 3130
3131 3131 def setNextFile(self):
3132 3132
3133 3133 if self.online:
3134 3134 newFile = self.__setNextFileOnline()
3135 3135 else:
3136 3136 newFile = self.__setNextFileOffline()
3137 3137
3138 3138 if not(newFile):
3139 3139 return 0
3140 3140
3141 3141 self.readHeader()
3142 3142
3143 3143 self.nReadBlocks = 0
3144 3144 # self.blockIndex = 1
3145 3145 return 1
3146 3146
3147 3147 def __searchFilesOffLine(self,
3148 3148 path,
3149 3149 startDate,
3150 3150 endDate,
3151 3151 startTime=datetime.time(0,0,0),
3152 3152 endTime=datetime.time(23,59,59),
3153 3153 set=None,
3154 3154 expLabel='',
3155 3155 ext='.fits',
3156 3156 walk=True):
3157 3157
3158 3158 pathList = []
3159 3159
3160 3160 if not walk:
3161 3161 pathList.append(path)
3162 3162
3163 3163 else:
3164 3164 dirList = []
3165 3165 for thisPath in os.listdir(path):
3166 3166 if not os.path.isdir(os.path.join(path,thisPath)):
3167 3167 continue
3168 3168 if not isDoyFolder(thisPath):
3169 3169 continue
3170 3170
3171 3171 dirList.append(thisPath)
3172 3172
3173 3173 if not(dirList):
3174 3174 return None, None
3175 3175
3176 3176 thisDate = startDate
3177 3177
3178 3178 while(thisDate <= endDate):
3179 3179 year = thisDate.timetuple().tm_year
3180 3180 doy = thisDate.timetuple().tm_yday
3181 3181
3182 3182 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
3183 3183 if len(matchlist) == 0:
3184 3184 thisDate += datetime.timedelta(1)
3185 3185 continue
3186 3186 for match in matchlist:
3187 3187 pathList.append(os.path.join(path,match,expLabel))
3188 3188
3189 3189 thisDate += datetime.timedelta(1)
3190 3190
3191 3191 if pathList == []:
3192 3192 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
3193 3193 return None, None
3194 3194
3195 3195 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
3196 3196
3197 3197 filenameList = []
3198 3198 datetimeList = []
3199 3199
3200 3200 for i in range(len(pathList)):
3201 3201
3202 3202 thisPath = pathList[i]
3203 3203
3204 3204 fileList = glob.glob1(thisPath, "*%s" %ext)
3205 3205 fileList.sort()
3206 3206
3207 3207 for file in fileList:
3208 3208
3209 3209 filename = os.path.join(thisPath,file)
3210 3210 thisDatetime = self.isFileinThisTime(filename, startTime, endTime)
3211 3211
3212 3212 if not(thisDatetime):
3213 3213 continue
3214 3214
3215 3215 filenameList.append(filename)
3216 3216 datetimeList.append(thisDatetime)
3217 3217
3218 3218 if not(filenameList):
3219 3219 print "Any file was found for the time range %s - %s" %(startTime, endTime)
3220 3220 return None, None
3221 3221
3222 3222 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
3223 3223 print
3224 3224
3225 3225 for i in range(len(filenameList)):
3226 3226 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
3227 3227
3228 3228 self.filenameList = filenameList
3229 3229 self.datetimeList = datetimeList
3230 3230
3231 3231 return pathList, filenameList
3232 3232
3233 3233 def setup(self, path=None,
3234 3234 startDate=None,
3235 3235 endDate=None,
3236 3236 startTime=datetime.time(0,0,0),
3237 3237 endTime=datetime.time(23,59,59),
3238 3238 set=0,
3239 3239 expLabel = "",
3240 3240 ext = None,
3241 3241 online = False,
3242 3242 delay = 60,
3243 3243 walk = True):
3244 3244
3245 3245 if path == None:
3246 3246 raise ValueError, "The path is not valid"
3247 3247
3248 3248 if ext == None:
3249 3249 ext = self.ext
3250 3250
3251 3251 if not(online):
3252 3252 print "Searching files in offline mode ..."
3253 3253 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
3254 3254 startTime=startTime, endTime=endTime,
3255 3255 set=set, expLabel=expLabel, ext=ext,
3256 3256 walk=walk)
3257 3257
3258 3258 if not(pathList):
3259 3259 print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
3260 3260 datetime.datetime.combine(startDate,startTime).ctime(),
3261 3261 datetime.datetime.combine(endDate,endTime).ctime())
3262 3262
3263 3263 sys.exit(-1)
3264 3264
3265 3265 self.fileIndex = -1
3266 3266 self.pathList = pathList
3267 3267 self.filenameList = filenameList
3268 3268
3269 3269 self.online = online
3270 3270 self.delay = delay
3271 3271 ext = ext.lower()
3272 3272 self.ext = ext
3273 3273
3274 3274 if not(self.setNextFile()):
3275 3275 if (startDate!=None) and (endDate!=None):
3276 3276 print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
3277 3277 elif startDate != None:
3278 3278 print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
3279 3279 else:
3280 3280 print "No files"
3281 3281
3282 3282 sys.exit(-1)
3283 3283
3284 3284
3285 3285
3286 3286 def readBlock(self):
3287 3287 dataObj = self.fitsObj[self.blockIndex]
3288 3288
3289 3289 self.data = dataObj.data
3290 3290 self.data_header_dict = dataObj.header
3291 3291 self.utc = self.data_header_dict['UTCTIME']
3292 3292
3293 3293 self.flagIsNewFile = 0
3294 3294 self.blockIndex += 1
3295 3295 self.nTotalBlocks += 1
3296 3296 self.nReadBlocks += 1
3297 3297
3298 3298 return 1
3299 3299
3300 3300 def __jumpToLastBlock(self):
3301 3301 raise ValueError, "No implemented"
3302 3302
3303 3303 def __waitNewBlock(self):
3304 3304 """
3305 3305 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
3306 3306
3307 3307 Si el modo de lectura es OffLine siempre retorn 0
3308 3308 """
3309 3309 if not self.online:
3310 3310 return 0
3311 3311
3312 3312 if (self.nReadBlocks >= self.dataBlocksPerFile):
3313 3313 return 0
3314 3314
3315 3315 currentPointer = self.fp.tell()
3316 3316
3317 3317 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
3318 3318
3319 3319 for nTries in range( self.nTries ):
3320 3320
3321 3321 self.fp.close()
3322 3322 self.fp = open( self.filename, 'rb' )
3323 3323 self.fp.seek( currentPointer )
3324 3324
3325 3325 self.fileSize = os.path.getsize( self.filename )
3326 3326 currentSize = self.fileSize - currentPointer
3327 3327
3328 3328 if ( currentSize >= neededSize ):
3329 3329 self.__rdBasicHeader()
3330 3330 return 1
3331 3331
3332 3332 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
3333 3333 time.sleep( self.delay )
3334 3334
3335 3335
3336 3336 return 0
3337 3337
3338 3338 def __setNewBlock(self):
3339 3339
3340 3340 if self.online:
3341 3341 self.__jumpToLastBlock()
3342 3342
3343 3343 if self.flagIsNewFile:
3344 3344 return 1
3345 3345
3346 3346 self.lastUTTime = self.utc
3347 3347
3348 3348 if self.online:
3349 3349 if self.__waitNewBlock():
3350 3350 return 1
3351 3351
3352 3352 if self.nReadBlocks < self.dataBlocksPerFile:
3353 3353 return 1
3354 3354
3355 3355 if not(self.setNextFile()):
3356 3356 return 0
3357 3357
3358 3358 deltaTime = self.utc - self.lastUTTime
3359 3359
3360 3360 self.flagTimeBlock = 0
3361 3361
3362 3362 if deltaTime > self.maxTimeStep:
3363 3363 self.flagTimeBlock = 1
3364 3364
3365 3365 return 1
3366 3366
3367 3367
3368 3368 def readNextBlock(self):
3369 3369 if not(self.__setNewBlock()):
3370 3370 return 0
3371 3371
3372 3372 if not(self.readBlock()):
3373 3373 return 0
3374 3374
3375 3375 return 1
3376 3376
3377 3377
3378 3378 def getData(self):
3379 3379
3380 3380 if self.flagNoMoreFiles:
3381 3381 self.dataOut.flagNoData = True
3382 3382 print 'Process finished'
3383 3383 return 0
3384 3384
3385 3385 self.flagTimeBlock = 0
3386 3386 self.flagIsNewBlock = 0
3387 3387
3388 3388 if not(self.readNextBlock()):
3389 3389 return 0
3390 3390
3391 3391 if self.data == None:
3392 3392 self.dataOut.flagNoData = True
3393 3393 return 0
3394 3394
3395 3395 self.dataOut.data = self.data
3396 3396 self.dataOut.data_header = self.data_header_dict
3397 3397 self.dataOut.utctime = self.utc
3398 3398
3399 3399 self.dataOut.header = self.header_dict
3400 3400 self.dataOut.expName = self.expName
3401 3401 self.dataOut.nChannels = self.nChannels
3402 3402 self.dataOut.timeZone = self.timeZone
3403 3403 self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
3404 3404 self.dataOut.comments = self.comments
3405 3405 self.dataOut.timeInterval = self.timeInterval
3406 3406 self.dataOut.channelList = self.channelList
3407 3407 self.dataOut.heightList = self.heightList
3408 3408 self.dataOut.flagNoData = False
3409 3409
3410 3410 return self.dataOut.data
3411 3411
3412 3412 def run(self, **kwargs):
3413 3413
3414 3414 if not(self.isConfig):
3415 3415 self.setup(**kwargs)
3416 3416 self.isConfig = True
3417 3417
3418 3418 self.getData() No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,535 +1,535
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
5 5 '''
6 6 import sys
7 7 import numpy
8 8 import copy
9 9 import datetime
10 10
11 11 class Header:
12 12
13 13 def __init__(self):
14 14 raise
15 15
16 16 def copy(self):
17 17 return copy.deepcopy(self)
18 18
19 19 def read():
20 20 pass
21 21
22 22 def write():
23 23 pass
24 24
25 25 def printInfo(self):
26 26
27 27 print "#"*100
28 28 print self.__class__.__name__.upper()
29 29 print "#"*100
30 30 for key in self.__dict__.keys():
31 31 print "%s = %s" %(key, self.__dict__[key])
32 32
33 33 class BasicHeader(Header):
34 34
35 35 size = None
36 36 version = None
37 37 dataBlock = None
38 38 utc = None
39 39 ltc = None
40 40 miliSecond = None
41 41 timeZone = None
42 42 dstFlag = None
43 43 errorCount = None
44 44 struct = None
45 45 datatime = None
46 46
47 47 __LOCALTIME = None
48 48
49 49 def __init__(self, useLocalTime=True):
50 50
51 51 self.size = 0
52 52 self.version = 0
53 53 self.dataBlock = 0
54 54 self.utc = 0
55 55 self.miliSecond = 0
56 56 self.timeZone = 0
57 57 self.dstFlag = 0
58 58 self.errorCount = 0
59 59 self.struct = numpy.dtype([
60 60 ('nSize','<u4'),
61 61 ('nVersion','<u2'),
62 62 ('nDataBlockId','<u4'),
63 63 ('nUtime','<u4'),
64 64 ('nMilsec','<u2'),
65 65 ('nTimezone','<i2'),
66 66 ('nDstflag','<i2'),
67 67 ('nErrorCount','<u4')
68 68 ])
69 69
70 70 self.useLocalTime = useLocalTime
71 71
72 72 def read(self, fp):
73 73 try:
74 74 header = numpy.fromfile(fp, self.struct,1)
75 75 self.size = int(header['nSize'][0])
76 76 self.version = int(header['nVersion'][0])
77 77 self.dataBlock = int(header['nDataBlockId'][0])
78 78 self.utc = int(header['nUtime'][0])
79 79 self.miliSecond = int(header['nMilsec'][0])
80 80 self.timeZone = int(header['nTimezone'][0])
81 81 self.dstFlag = int(header['nDstflag'][0])
82 82 self.errorCount = int(header['nErrorCount'][0])
83 83
84 84 self.ltc = self.utc
85 85
86 86 if self.useLocalTime:
87 87 self.ltc -= self.timeZone*60
88 88
89 89 self.datatime = datetime.datetime.utcfromtimestamp(self.ltc)
90 90
91 91 except Exception, e:
92 92 print "BasicHeader: "
93 93 print e
94 94 return 0
95 95
96 96 return 1
97 97
98 98 def write(self, fp):
99 99
100 100 headerTuple = (self.size,self.version,self.dataBlock,self.utc,self.miliSecond,self.timeZone,self.dstFlag,self.errorCount)
101 101 header = numpy.array(headerTuple,self.struct)
102 102 header.tofile(fp)
103 103
104 104 return 1
105 105
106 106 class SystemHeader(Header):
107 107
108 108 size = None
109 109 nSamples = None
110 110 nProfiles = None
111 111 nChannels = None
112 112 adcResolution = None
113 113 pciDioBusWidth = None
114 114 struct = None
115 115
116 116 def __init__(self):
117 117 self.size = 0
118 118 self.nSamples = 0
119 119 self.nProfiles = 0
120 120 self.nChannels = 0
121 121 self.adcResolution = 0
122 122 self.pciDioBusWidth = 0
123 123 self.struct = numpy.dtype([
124 124 ('nSize','<u4'),
125 125 ('nNumSamples','<u4'),
126 126 ('nNumProfiles','<u4'),
127 127 ('nNumChannels','<u4'),
128 128 ('nADCResolution','<u4'),
129 129 ('nPCDIOBusWidth','<u4'),
130 130 ])
131 131
132 132
133 133 def read(self, fp):
134 134 try:
135 135 header = numpy.fromfile(fp,self.struct,1)
136 136 self.size = header['nSize'][0]
137 137 self.nSamples = header['nNumSamples'][0]
138 138 self.nProfiles = header['nNumProfiles'][0]
139 139 self.nChannels = header['nNumChannels'][0]
140 140 self.adcResolution = header['nADCResolution'][0]
141 141 self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
142 142
143 143 except Exception, e:
144 144 print "SystemHeader: " + e
145 145 return 0
146 146
147 147 return 1
148 148
149 149 def write(self, fp):
150 150 headerTuple = (self.size,self.nSamples,self.nProfiles,self.nChannels,self.adcResolution,self.pciDioBusWidth)
151 151 header = numpy.array(headerTuple,self.struct)
152 152 header.tofile(fp)
153 153
154 154 return 1
155 155
156 156 class RadarControllerHeader(Header):
157 157
158 158 size = None
159 159 expType = None
160 160 nTx = None
161 161 ipp = None
162 162 txA = None
163 163 txB = None
164 164 nWindows = None
165 165 numTaus = None
166 166 codeType = None
167 167 line6Function = None
168 168 line5Function = None
169 169 fClock = None
170 170 prePulseBefore = None
171 171 prePulserAfter = None
172 172 rangeIpp = None
173 173 rangeTxA = None
174 174 rangeTxB = None
175 175 struct = None
176 176
177 177 def __init__(self):
178 178 self.size = 0
179 179 self.expType = 0
180 180 self.nTx = 0
181 181 self.ipp = 0
182 182 self.txA = 0
183 183 self.txB = 0
184 184 self.nWindows = 0
185 185 self.numTaus = 0
186 186 self.codeType = 0
187 187 self.line6Function = 0
188 188 self.line5Function = 0
189 189 self.fClock = 0
190 190 self.prePulseBefore = 0
191 191 self.prePulserAfter = 0
192 192 self.rangeIpp = 0
193 193 self.rangeTxA = 0
194 194 self.rangeTxB = 0
195 195 self.struct = numpy.dtype([
196 196 ('nSize','<u4'),
197 197 ('nExpType','<u4'),
198 198 ('nNTx','<u4'),
199 199 ('fIpp','<f4'),
200 200 ('fTxA','<f4'),
201 201 ('fTxB','<f4'),
202 202 ('nNumWindows','<u4'),
203 203 ('nNumTaus','<u4'),
204 204 ('nCodeType','<u4'),
205 205 ('nLine6Function','<u4'),
206 206 ('nLine5Function','<u4'),
207 207 ('fClock','<f4'),
208 208 ('nPrePulseBefore','<u4'),
209 209 ('nPrePulseAfter','<u4'),
210 210 ('sRangeIPP','<a20'),
211 211 ('sRangeTxA','<a20'),
212 212 ('sRangeTxB','<a20'),
213 213 ])
214 214
215 215 self.samplingWindowStruct = numpy.dtype([('h0','<f4'),('dh','<f4'),('nsa','<u4')])
216 216
217 217 self.samplingWindow = None
218 218 self.nHeights = None
219 219 self.firstHeight = None
220 220 self.deltaHeight = None
221 221 self.samplesWin = None
222 222
223 223 self.nCode = None
224 224 self.nBaud = None
225 225 self.code = None
226 226 self.flip1 = None
227 227 self.flip2 = None
228 228
229 229 self.dynamic = numpy.array([],numpy.dtype('byte'))
230 230
231 231
232 232 def read(self, fp):
233 233 try:
234 234 startFp = fp.tell()
235 235 header = numpy.fromfile(fp,self.struct,1)
236 236 self.size = int(header['nSize'][0])
237 237 self.expType = int(header['nExpType'][0])
238 238 self.nTx = int(header['nNTx'][0])
239 239 self.ipp = float(header['fIpp'][0])
240 240 self.txA = float(header['fTxA'][0])
241 241 self.txB = float(header['fTxB'][0])
242 242 self.nWindows = int(header['nNumWindows'][0])
243 243 self.numTaus = int(header['nNumTaus'][0])
244 244 self.codeType = int(header['nCodeType'][0])
245 245 self.line6Function = int(header['nLine6Function'][0])
246 246 self.line5Function = int(header['nLine5Function'][0])
247 247 self.fClock = float(header['fClock'][0])
248 248 self.prePulseBefore = int(header['nPrePulseBefore'][0])
249 249 self.prePulserAfter = int(header['nPrePulseAfter'][0])
250 250 self.rangeIpp = header['sRangeIPP'][0]
251 251 self.rangeTxA = header['sRangeTxA'][0]
252 252 self.rangeTxB = header['sRangeTxB'][0]
253 253 # jump Dynamic Radar Controller Header
254 254 jumpFp = self.size - 116
255 255 self.dynamic = numpy.fromfile(fp,numpy.dtype('byte'),jumpFp)
256 256 #pointer backward to dynamic header and read
257 257 backFp = fp.tell() - jumpFp
258 258 fp.seek(backFp)
259 259
260 260 self.samplingWindow = numpy.fromfile(fp,self.samplingWindowStruct,self.nWindows)
261 261 self.nHeights = int(numpy.sum(self.samplingWindow['nsa']))
262 262 self.firstHeight = self.samplingWindow['h0']
263 263 self.deltaHeight = self.samplingWindow['dh']
264 264 self.samplesWin = self.samplingWindow['nsa']
265 265
266 266 self.Taus = numpy.fromfile(fp,'<f4',self.numTaus)
267 267
268 268 if self.codeType != 0:
269 269 self.nCode = int(numpy.fromfile(fp,'<u4',1))
270 270 self.nBaud = int(numpy.fromfile(fp,'<u4',1))
271 271 self.code = numpy.empty([self.nCode,self.nBaud],dtype='u1')
272 272
273 273 for ic in range(self.nCode):
274 274 temp = numpy.fromfile(fp,'u4',int(numpy.ceil(self.nBaud/32.)))
275 275 for ib in range(self.nBaud-1,-1,-1):
276 276 self.code[ic,ib] = temp[ib/32]%2
277 277 temp[ib/32] = temp[ib/32]/2
278 278 self.code = 2.0*self.code - 1.0
279 279
280 280 if self.line5Function == RCfunction.FLIP:
281 281 self.flip1 = numpy.fromfile(fp,'<u4',1)
282 282
283 283 if self.line6Function == RCfunction.FLIP:
284 284 self.flip2 = numpy.fromfile(fp,'<u4',1)
285 285
286 286 endFp = self.size + startFp
287 287 jumpFp = endFp - fp.tell()
288 288 if jumpFp > 0:
289 289 fp.seek(jumpFp)
290 290
291 291 except Exception, e:
292 292 print "RadarControllerHeader: " + e
293 293 return 0
294 294
295 295 return 1
296 296
297 297 def write(self, fp):
298 298 headerTuple = (self.size,
299 299 self.expType,
300 300 self.nTx,
301 301 self.ipp,
302 302 self.txA,
303 303 self.txB,
304 304 self.nWindows,
305 305 self.numTaus,
306 306 self.codeType,
307 307 self.line6Function,
308 308 self.line5Function,
309 309 self.fClock,
310 310 self.prePulseBefore,
311 311 self.prePulserAfter,
312 312 self.rangeIpp,
313 313 self.rangeTxA,
314 314 self.rangeTxB)
315 315
316 316 header = numpy.array(headerTuple,self.struct)
317 317 header.tofile(fp)
318 318
319 319 dynamic = self.dynamic
320 320 dynamic.tofile(fp)
321 321
322 322 return 1
323 323
324 324
325 325
326 326 class ProcessingHeader(Header):
327 327
328 328 size = None
329 329 dtype = None
330 330 blockSize = None
331 331 profilesPerBlock = None
332 332 dataBlocksPerFile = None
333 333 nWindows = None
334 334 processFlags = None
335 335 nCohInt = None
336 336 nIncohInt = None
337 337 totalSpectra = None
338 338 struct = None
339 339 flag_dc = None
340 340 flag_cspc = None
341 341
342 342 def __init__(self):
343 343 self.size = 0
344 344 self.dtype = 0
345 345 self.blockSize = 0
346 346 self.profilesPerBlock = 0
347 347 self.dataBlocksPerFile = 0
348 348 self.nWindows = 0
349 349 self.processFlags = 0
350 350 self.nCohInt = 0
351 351 self.nIncohInt = 0
352 352 self.totalSpectra = 0
353 353 self.struct = numpy.dtype([
354 354 ('nSize','<u4'),
355 355 ('nDataType','<u4'),
356 356 ('nSizeOfDataBlock','<u4'),
357 357 ('nProfilesperBlock','<u4'),
358 358 ('nDataBlocksperFile','<u4'),
359 359 ('nNumWindows','<u4'),
360 360 ('nProcessFlags','<u4'),
361 361 ('nCoherentIntegrations','<u4'),
362 362 ('nIncoherentIntegrations','<u4'),
363 363 ('nTotalSpectra','<u4')
364 364 ])
365 365 self.samplingWindow = 0
366 366 self.structSamplingWindow = numpy.dtype([('h0','<f4'),('dh','<f4'),('nsa','<u4')])
367 367 self.nHeights = 0
368 368 self.firstHeight = 0
369 369 self.deltaHeight = 0
370 370 self.samplesWin = 0
371 371 self.spectraComb = 0
372 self.nCode = None
373 self.code = None
374 self.nBaud = None
372 # self.nCode = None
373 # self.code = None
374 # self.nBaud = None
375 375 self.shif_fft = False
376 376 self.flag_dc = False
377 377 self.flag_cspc = False
378 378
379 379 def read(self, fp):
380 380 # try:
381 381 header = numpy.fromfile(fp,self.struct,1)
382 382 self.size = int(header['nSize'][0])
383 383 self.dtype = int(header['nDataType'][0])
384 384 self.blockSize = int(header['nSizeOfDataBlock'][0])
385 385 self.profilesPerBlock = int(header['nProfilesperBlock'][0])
386 386 self.dataBlocksPerFile = int(header['nDataBlocksperFile'][0])
387 387 self.nWindows = int(header['nNumWindows'][0])
388 388 self.processFlags = header['nProcessFlags']
389 389 self.nCohInt = int(header['nCoherentIntegrations'][0])
390 390 self.nIncohInt = int(header['nIncoherentIntegrations'][0])
391 391 self.totalSpectra = int(header['nTotalSpectra'][0])
392 392 self.samplingWindow = numpy.fromfile(fp,self.structSamplingWindow,self.nWindows)
393 393 self.nHeights = int(numpy.sum(self.samplingWindow['nsa']))
394 394 self.firstHeight = float(self.samplingWindow['h0'][0])
395 395 self.deltaHeight = float(self.samplingWindow['dh'][0])
396 396 self.samplesWin = self.samplingWindow['nsa']
397 397 self.spectraComb = numpy.fromfile(fp,'u1',2*self.totalSpectra)
398 398
399 399 # if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
400 400 # self.nCode = int(numpy.fromfile(fp,'<u4',1))
401 401 # self.nBaud = int(numpy.fromfile(fp,'<u4',1))
402 402 # self.code = numpy.fromfile(fp,'<f4',self.nCode*self.nBaud).reshape(self.nCode,self.nBaud)
403 403
404 404 if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA):
405 405 self.shif_fft = True
406 406 else:
407 407 self.shif_fft = False
408 408
409 409 if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC):
410 410 self.flag_dc = True
411 411
412 412 nChannels = 0
413 413 nPairs = 0
414 414 pairList = []
415 415
416 416 for i in range( 0, self.totalSpectra*2, 2 ):
417 417 if self.spectraComb[i] == self.spectraComb[i+1]:
418 418 nChannels = nChannels + 1 #par de canales iguales
419 419 else:
420 420 nPairs = nPairs + 1 #par de canales diferentes
421 421 pairList.append( (self.spectraComb[i], self.spectraComb[i+1]) )
422 422
423 423 self.flag_cspc = False
424 424 if nPairs > 0:
425 425 self.flag_cspc = True
426 426
427 427 # except Exception, e:
428 428 # print "Error ProcessingHeader: "
429 429 # return 0
430 430
431 431 return 1
432 432
433 433 def write(self, fp):
434 434 headerTuple = (self.size,
435 435 self.dtype,
436 436 self.blockSize,
437 437 self.profilesPerBlock,
438 438 self.dataBlocksPerFile,
439 439 self.nWindows,
440 440 self.processFlags,
441 441 self.nCohInt,
442 442 self.nIncohInt,
443 443 self.totalSpectra)
444 444
445 445 header = numpy.array(headerTuple,self.struct)
446 446 header.tofile(fp)
447 447
448 448 if self.nWindows != 0:
449 449 sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin)
450 450 samplingWindow = numpy.array(sampleWindowTuple,self.structSamplingWindow)
451 451 samplingWindow.tofile(fp)
452 452
453 453
454 454 if self.totalSpectra != 0:
455 455 spectraComb = numpy.array([],numpy.dtype('u1'))
456 456 spectraComb = self.spectraComb
457 457 spectraComb.tofile(fp)
458 458
459 459 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
460 460 # nCode = numpy.array([self.nCode], numpy.dtype('u4')) #Probar con un dato que almacene codigo, hasta el momento no se hizo la prueba
461 461 # nCode.tofile(fp)
462 462 #
463 463 # nBaud = numpy.array([self.nBaud], numpy.dtype('u4'))
464 464 # nBaud.tofile(fp)
465 465 #
466 466 # code = self.code.reshape(self.nCode*self.nBaud)
467 467 # code = code.astype(numpy.dtype('<f4'))
468 468 # code.tofile(fp)
469 469
470 470 return 1
471 471
472 472 class RCfunction:
473 473 NONE=0
474 474 FLIP=1
475 475 CODE=2
476 476 SAMPLING=3
477 477 LIN6DIV256=4
478 478 SYNCHRO=5
479 479
480 480 class nCodeType:
481 481 NONE=0
482 482 USERDEFINE=1
483 483 BARKER2=2
484 484 BARKER3=3
485 485 BARKER4=4
486 486 BARKER5=5
487 487 BARKER7=6
488 488 BARKER11=7
489 489 BARKER13=8
490 490 AC128=9
491 491 COMPLEMENTARYCODE2=10
492 492 COMPLEMENTARYCODE4=11
493 493 COMPLEMENTARYCODE8=12
494 494 COMPLEMENTARYCODE16=13
495 495 COMPLEMENTARYCODE32=14
496 496 COMPLEMENTARYCODE64=15
497 497 COMPLEMENTARYCODE128=16
498 498 CODE_BINARY28=17
499 499
500 500 class PROCFLAG:
501 501 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
502 502 DECODE_DATA = numpy.uint32(0x00000002)
503 503 SPECTRA_CALC = numpy.uint32(0x00000004)
504 504 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
505 505 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
506 506 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
507 507
508 508 DATATYPE_CHAR = numpy.uint32(0x00000040)
509 509 DATATYPE_SHORT = numpy.uint32(0x00000080)
510 510 DATATYPE_LONG = numpy.uint32(0x00000100)
511 511 DATATYPE_INT64 = numpy.uint32(0x00000200)
512 512 DATATYPE_FLOAT = numpy.uint32(0x00000400)
513 513 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
514 514
515 515 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
516 516 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
517 517 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
518 518
519 519 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
520 520 DEFLIP_DATA = numpy.uint32(0x00010000)
521 521 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
522 522
523 523 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
524 524 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
525 525 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
526 526 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
527 527 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
528 528
529 529 EXP_NAME_ESP = numpy.uint32(0x00200000)
530 530 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
531 531
532 532 OPERATION_MASK = numpy.uint32(0x0000003F)
533 533 DATATYPE_MASK = numpy.uint32(0x00000FC0)
534 534 DATAARRANGE_MASK = numpy.uint32(0x00007000)
535 535 ACQ_SYS_MASK = numpy.uint32(0x001C0000) No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,2039 +1,2040
1 1 '''
2 2
3 3 $Author: dsuarez $
4 4 $Id: Processor.py 1 2012-11-12 18:56:07Z dsuarez $
5 5 '''
6 6 import os
7 7 import numpy
8 8 import datetime
9 9 import time
10 10 import math
11 11 from jrodata import *
12 12 from jrodataIO import *
13 13 from jroplot import *
14 14
15 15 try:
16 16 import cfunctions
17 17 except:
18 18 pass
19 19
20 20 class ProcessingUnit:
21 21
22 22 """
23 23 Esta es la clase base para el procesamiento de datos.
24 24
25 25 Contiene el metodo "call" para llamar operaciones. Las operaciones pueden ser:
26 26 - Metodos internos (callMethod)
27 27 - Objetos del tipo Operation (callObject). Antes de ser llamados, estos objetos
28 28 tienen que ser agreagados con el metodo "add".
29 29
30 30 """
31 31 # objeto de datos de entrada (Voltage, Spectra o Correlation)
32 32 dataIn = None
33 33
34 34 # objeto de datos de entrada (Voltage, Spectra o Correlation)
35 35 dataOut = None
36 36
37 37
38 38 objectDict = None
39 39
40 40 def __init__(self):
41 41
42 42 self.objectDict = {}
43 43
44 44 def init(self):
45 45
46 46 raise ValueError, "Not implemented"
47 47
48 48 def addOperation(self, object, objId):
49 49
50 50 """
51 51 Agrega el objeto "object" a la lista de objetos "self.objectList" y retorna el
52 52 identificador asociado a este objeto.
53 53
54 54 Input:
55 55
56 56 object : objeto de la clase "Operation"
57 57
58 58 Return:
59 59
60 60 objId : identificador del objeto, necesario para ejecutar la operacion
61 61 """
62 62
63 63 self.objectDict[objId] = object
64 64
65 65 return objId
66 66
67 67 def operation(self, **kwargs):
68 68
69 69 """
70 70 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
71 71 atributos del objeto dataOut
72 72
73 73 Input:
74 74
75 75 **kwargs : Diccionario de argumentos de la funcion a ejecutar
76 76 """
77 77
78 78 raise ValueError, "ImplementedError"
79 79
80 80 def callMethod(self, name, **kwargs):
81 81
82 82 """
83 83 Ejecuta el metodo con el nombre "name" y con argumentos **kwargs de la propia clase.
84 84
85 85 Input:
86 86 name : nombre del metodo a ejecutar
87 87
88 88 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
89 89
90 90 """
91 91 if name != 'run':
92 92
93 93 if name == 'init' and self.dataIn.isEmpty():
94 94 self.dataOut.flagNoData = True
95 95 return False
96 96
97 97 if name != 'init' and self.dataOut.isEmpty():
98 98 return False
99 99
100 100 methodToCall = getattr(self, name)
101 101
102 102 methodToCall(**kwargs)
103 103
104 104 if name != 'run':
105 105 return True
106 106
107 107 if self.dataOut.isEmpty():
108 108 return False
109 109
110 110 return True
111 111
112 112 def callObject(self, objId, **kwargs):
113 113
114 114 """
115 115 Ejecuta la operacion asociada al identificador del objeto "objId"
116 116
117 117 Input:
118 118
119 119 objId : identificador del objeto a ejecutar
120 120
121 121 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
122 122
123 123 Return:
124 124
125 125 None
126 126 """
127 127
128 128 if self.dataOut.isEmpty():
129 129 return False
130 130
131 131 object = self.objectDict[objId]
132 132
133 133 object.run(self.dataOut, **kwargs)
134 134
135 135 return True
136 136
137 137 def call(self, operationConf, **kwargs):
138 138
139 139 """
140 140 Return True si ejecuta la operacion "operationConf.name" con los
141 141 argumentos "**kwargs". False si la operacion no se ha ejecutado.
142 142 La operacion puede ser de dos tipos:
143 143
144 144 1. Un metodo propio de esta clase:
145 145
146 146 operation.type = "self"
147 147
148 148 2. El metodo "run" de un objeto del tipo Operation o de un derivado de ella:
149 149 operation.type = "other".
150 150
151 151 Este objeto de tipo Operation debe de haber sido agregado antes con el metodo:
152 152 "addOperation" e identificado con el operation.id
153 153
154 154
155 155 con el id de la operacion.
156 156
157 157 Input:
158 158
159 159 Operation : Objeto del tipo operacion con los atributos: name, type y id.
160 160
161 161 """
162 162
163 163 if operationConf.type == 'self':
164 164 sts = self.callMethod(operationConf.name, **kwargs)
165 165
166 166 if operationConf.type == 'other':
167 167 sts = self.callObject(operationConf.id, **kwargs)
168 168
169 169 return sts
170 170
171 171 def setInput(self, dataIn):
172 172
173 173 self.dataIn = dataIn
174 174
175 175 def getOutput(self):
176 176
177 177 return self.dataOut
178 178
179 179 class Operation():
180 180
181 181 """
182 182 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
183 183 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
184 184 acumulacion dentro de esta clase
185 185
186 186 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
187 187
188 188 """
189 189
190 190 __buffer = None
191 191 __isConfig = False
192 192
193 193 def __init__(self):
194 194
195 195 pass
196 196
197 197 def run(self, dataIn, **kwargs):
198 198
199 199 """
200 200 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los atributos del objeto dataIn.
201 201
202 202 Input:
203 203
204 204 dataIn : objeto del tipo JROData
205 205
206 206 Return:
207 207
208 208 None
209 209
210 210 Affected:
211 211 __buffer : buffer de recepcion de datos.
212 212
213 213 """
214 214
215 215 raise ValueError, "ImplementedError"
216 216
217 217 class VoltageProc(ProcessingUnit):
218 218
219 219
220 220 def __init__(self):
221 221
222 222 self.objectDict = {}
223 223 self.dataOut = Voltage()
224 224 self.flip = 1
225 225
226 226 def init(self):
227 227
228 228 self.dataOut.copy(self.dataIn)
229 229 # No necesita copiar en cada init() los atributos de dataIn
230 230 # la copia deberia hacerse por cada nuevo bloque de datos
231 231
232 232 def selectChannels(self, channelList):
233 233
234 234 channelIndexList = []
235 235
236 236 for channel in channelList:
237 237 index = self.dataOut.channelList.index(channel)
238 238 channelIndexList.append(index)
239 239
240 240 self.selectChannelsByIndex(channelIndexList)
241 241
242 242 def selectChannelsByIndex(self, channelIndexList):
243 243 """
244 244 Selecciona un bloque de datos en base a canales segun el channelIndexList
245 245
246 246 Input:
247 247 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
248 248
249 249 Affected:
250 250 self.dataOut.data
251 251 self.dataOut.channelIndexList
252 252 self.dataOut.nChannels
253 253 self.dataOut.m_ProcessingHeader.totalSpectra
254 254 self.dataOut.systemHeaderObj.numChannels
255 255 self.dataOut.m_ProcessingHeader.blockSize
256 256
257 257 Return:
258 258 None
259 259 """
260 260
261 261 for channelIndex in channelIndexList:
262 262 if channelIndex not in self.dataOut.channelIndexList:
263 263 print channelIndexList
264 264 raise ValueError, "The value %d in channelIndexList is not valid" %channelIndex
265 265
266 266 nChannels = len(channelIndexList)
267 267
268 268 data = self.dataOut.data[channelIndexList,:]
269 269
270 270 self.dataOut.data = data
271 271 self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
272 272 # self.dataOut.nChannels = nChannels
273 273
274 274 return 1
275 275
276 276 def selectHeights(self, minHei=None, maxHei=None):
277 277 """
278 278 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
279 279 minHei <= height <= maxHei
280 280
281 281 Input:
282 282 minHei : valor minimo de altura a considerar
283 283 maxHei : valor maximo de altura a considerar
284 284
285 285 Affected:
286 286 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
287 287
288 288 Return:
289 289 1 si el metodo se ejecuto con exito caso contrario devuelve 0
290 290 """
291 291
292 292 if minHei == None:
293 293 minHei = self.dataOut.heightList[0]
294 294
295 295 if maxHei == None:
296 296 maxHei = self.dataOut.heightList[-1]
297 297
298 298 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
299 299 raise ValueError, "some value in (%d,%d) is not valid" % (minHei, maxHei)
300 300
301 301
302 302 if (maxHei > self.dataOut.heightList[-1]):
303 303 maxHei = self.dataOut.heightList[-1]
304 304 # raise ValueError, "some value in (%d,%d) is not valid" % (minHei, maxHei)
305 305
306 306 minIndex = 0
307 307 maxIndex = 0
308 308 heights = self.dataOut.heightList
309 309
310 310 inda = numpy.where(heights >= minHei)
311 311 indb = numpy.where(heights <= maxHei)
312 312
313 313 try:
314 314 minIndex = inda[0][0]
315 315 except:
316 316 minIndex = 0
317 317
318 318 try:
319 319 maxIndex = indb[0][-1]
320 320 except:
321 321 maxIndex = len(heights)
322 322
323 323 self.selectHeightsByIndex(minIndex, maxIndex)
324 324
325 325 return 1
326 326
327 327
328 328 def selectHeightsByIndex(self, minIndex, maxIndex):
329 329 """
330 330 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
331 331 minIndex <= index <= maxIndex
332 332
333 333 Input:
334 334 minIndex : valor de indice minimo de altura a considerar
335 335 maxIndex : valor de indice maximo de altura a considerar
336 336
337 337 Affected:
338 338 self.dataOut.data
339 339 self.dataOut.heightList
340 340
341 341 Return:
342 342 1 si el metodo se ejecuto con exito caso contrario devuelve 0
343 343 """
344 344
345 345 if (minIndex < 0) or (minIndex > maxIndex):
346 346 raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
347 347
348 348 if (maxIndex >= self.dataOut.nHeights):
349 349 maxIndex = self.dataOut.nHeights-1
350 350 # raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
351 351
352 352 nHeights = maxIndex - minIndex + 1
353 353
354 354 #voltage
355 355 data = self.dataOut.data[:,minIndex:maxIndex+1]
356 356
357 357 firstHeight = self.dataOut.heightList[minIndex]
358 358
359 359 self.dataOut.data = data
360 360 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex+1]
361 361
362 362 return 1
363 363
364 364
365 365 def filterByHeights(self, window):
366 366 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
367 367
368 368 if window == None:
369 369 window = (self.dataOut.radarControllerHeaderObj.txA/self.dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
370 370
371 371 newdelta = deltaHeight * window
372 372 r = self.dataOut.data.shape[1] % window
373 373 buffer = self.dataOut.data[:,0:self.dataOut.data.shape[1]-r]
374 374 buffer = buffer.reshape(self.dataOut.data.shape[0],self.dataOut.data.shape[1]/window,window)
375 375 buffer = numpy.sum(buffer,2)
376 376 self.dataOut.data = buffer
377 377 self.dataOut.heightList = numpy.arange(self.dataOut.heightList[0],newdelta*(self.dataOut.nHeights-r)/window,newdelta)
378 378 self.dataOut.windowOfFilter = window
379 379
380 380 def deFlip(self):
381 381 self.dataOut.data *= self.flip
382 382 self.flip *= -1.
383 383
384 384 def setRadarFrequency(self, frequency=None):
385 385 if frequency != None:
386 386 self.dataOut.frequency = frequency
387 387
388 388 return 1
389 389
390 390 class CohInt(Operation):
391 391
392 392 __isConfig = False
393 393
394 394 __profIndex = 0
395 395 __withOverapping = False
396 396
397 397 __byTime = False
398 398 __initime = None
399 399 __lastdatatime = None
400 400 __integrationtime = None
401 401
402 402 __buffer = None
403 403
404 404 __dataReady = False
405 405
406 406 n = None
407 407
408 408
409 409 def __init__(self):
410 410
411 411 self.__isConfig = False
412 412
413 413 def setup(self, n=None, timeInterval=None, overlapping=False):
414 414 """
415 415 Set the parameters of the integration class.
416 416
417 417 Inputs:
418 418
419 419 n : Number of coherent integrations
420 420 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
421 421 overlapping :
422 422
423 423 """
424 424
425 425 self.__initime = None
426 426 self.__lastdatatime = 0
427 427 self.__buffer = None
428 428 self.__dataReady = False
429 429
430 430
431 431 if n == None and timeInterval == None:
432 432 raise ValueError, "n or timeInterval should be specified ..."
433 433
434 434 if n != None:
435 435 self.n = n
436 436 self.__byTime = False
437 437 else:
438 438 self.__integrationtime = timeInterval * 60. #if (type(timeInterval)!=integer) -> change this line
439 439 self.n = 9999
440 440 self.__byTime = True
441 441
442 442 if overlapping:
443 443 self.__withOverapping = True
444 444 self.__buffer = None
445 445 else:
446 446 self.__withOverapping = False
447 447 self.__buffer = 0
448 448
449 449 self.__profIndex = 0
450 450
451 451 def putData(self, data):
452 452
453 453 """
454 454 Add a profile to the __buffer and increase in one the __profileIndex
455 455
456 456 """
457 457
458 458 if not self.__withOverapping:
459 459 self.__buffer += data.copy()
460 460 self.__profIndex += 1
461 461 return
462 462
463 463 #Overlapping data
464 464 nChannels, nHeis = data.shape
465 465 data = numpy.reshape(data, (1, nChannels, nHeis))
466 466
467 467 #If the buffer is empty then it takes the data value
468 468 if self.__buffer == None:
469 469 self.__buffer = data
470 470 self.__profIndex += 1
471 471 return
472 472
473 473 #If the buffer length is lower than n then stakcing the data value
474 474 if self.__profIndex < self.n:
475 475 self.__buffer = numpy.vstack((self.__buffer, data))
476 476 self.__profIndex += 1
477 477 return
478 478
479 479 #If the buffer length is equal to n then replacing the last buffer value with the data value
480 480 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
481 481 self.__buffer[self.n-1] = data
482 482 self.__profIndex = self.n
483 483 return
484 484
485 485
486 486 def pushData(self):
487 487 """
488 488 Return the sum of the last profiles and the profiles used in the sum.
489 489
490 490 Affected:
491 491
492 492 self.__profileIndex
493 493
494 494 """
495 495
496 496 if not self.__withOverapping:
497 497 data = self.__buffer
498 498 n = self.__profIndex
499 499
500 500 self.__buffer = 0
501 501 self.__profIndex = 0
502 502
503 503 return data, n
504 504
505 505 #Integration with Overlapping
506 506 data = numpy.sum(self.__buffer, axis=0)
507 507 n = self.__profIndex
508 508
509 509 return data, n
510 510
511 511 def byProfiles(self, data):
512 512
513 513 self.__dataReady = False
514 514 avgdata = None
515 515 n = None
516 516
517 517 self.putData(data)
518 518
519 519 if self.__profIndex == self.n:
520 520
521 521 avgdata, n = self.pushData()
522 522 self.__dataReady = True
523 523
524 524 return avgdata
525 525
526 526 def byTime(self, data, datatime):
527 527
528 528 self.__dataReady = False
529 529 avgdata = None
530 530 n = None
531 531
532 532 self.putData(data)
533 533
534 534 if (datatime - self.__initime) >= self.__integrationtime:
535 535 avgdata, n = self.pushData()
536 536 self.n = n
537 537 self.__dataReady = True
538 538
539 539 return avgdata
540 540
541 541 def integrate(self, data, datatime=None):
542 542
543 543 if self.__initime == None:
544 544 self.__initime = datatime
545 545
546 546 if self.__byTime:
547 547 avgdata = self.byTime(data, datatime)
548 548 else:
549 549 avgdata = self.byProfiles(data)
550 550
551 551
552 552 self.__lastdatatime = datatime
553 553
554 554 if avgdata == None:
555 555 return None, None
556 556
557 557 avgdatatime = self.__initime
558 558
559 559 deltatime = datatime -self.__lastdatatime
560 560
561 561 if not self.__withOverapping:
562 562 self.__initime = datatime
563 563 else:
564 564 self.__initime += deltatime
565 565
566 566 return avgdata, avgdatatime
567 567
568 568 def run(self, dataOut, **kwargs):
569 569
570 570 if not self.__isConfig:
571 571 self.setup(**kwargs)
572 572 self.__isConfig = True
573 573
574 574 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
575 575
576 576 # dataOut.timeInterval *= n
577 577 dataOut.flagNoData = True
578 578
579 579 if self.__dataReady:
580 580 dataOut.data = avgdata
581 581 dataOut.nCohInt *= self.n
582 582 dataOut.utctime = avgdatatime
583 583 dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
584 584 dataOut.flagNoData = False
585 585
586 586
587 587 class Decoder(Operation):
588 588
589 589 __isConfig = False
590 590 __profIndex = 0
591 591
592 592 code = None
593 593
594 594 nCode = None
595 595 nBaud = None
596 596
597 597 def __init__(self):
598 598
599 599 self.__isConfig = False
600 600
601 601 def setup(self, code, shape):
602 602
603 603 self.__profIndex = 0
604 604
605 605 self.code = code
606 606
607 607 self.nCode = len(code)
608 608 self.nBaud = len(code[0])
609 609
610 610 self.__nChannels, self.__nHeis = shape
611 611
612 612 __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
613 613
614 614 __codeBuffer[:,0:self.nBaud] = self.code
615 615
616 616 self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
617 617
618 618 self.ndatadec = self.__nHeis - self.nBaud + 1
619 619
620 620 self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
621 621
622 622 def convolutionInFreq(self, data):
623 623
624 624 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
625 625
626 626 fft_data = numpy.fft.fft(data, axis=1)
627 627
628 628 conv = fft_data*fft_code
629 629
630 630 data = numpy.fft.ifft(conv,axis=1)
631 631
632 632 datadec = data[:,:-self.nBaud+1]
633 633
634 634 return datadec
635 635
636 636 def convolutionInFreqOpt(self, data):
637 637
638 638 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
639 639
640 640 data = cfunctions.decoder(fft_code, data)
641 641
642 642 datadec = data[:,:-self.nBaud+1]
643 643
644 644 return datadec
645 645
646 646 def convolutionInTime(self, data):
647 647
648 648 code = self.code[self.__profIndex]
649 649
650 650 for i in range(self.__nChannels):
651 651 self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='valid')
652 652
653 653 return self.datadecTime
654 654
655 655 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0):
656 656
657 if not self.__isConfig:
658
659 657 if code == None:
660 658 code = dataOut.code
661 659 else:
662 660 code = numpy.array(code).reshape(nCode,nBaud)
663 661 dataOut.code = code
664 662 dataOut.nCode = nCode
665 663 dataOut.nBaud = nBaud
664 dataOut.radarControllerHeaderObj.code = code
665 dataOut.radarControllerHeaderObj.nCode = nCode
666 dataOut.radarControllerHeaderObj.nBaud = nBaud
666 667
667 if code == None:
668 return 1
668
669 if not self.__isConfig:
669 670
670 671 self.setup(code, dataOut.data.shape)
671 672 self.__isConfig = True
672 673
673 674 if mode == 0:
674 675 datadec = self.convolutionInTime(dataOut.data)
675 676
676 677 if mode == 1:
677 678 datadec = self.convolutionInFreq(dataOut.data)
678 679
679 680 if mode == 2:
680 681 datadec = self.convolutionInFreqOpt(dataOut.data)
681 682
682 683 dataOut.data = datadec
683 684
684 685 dataOut.heightList = dataOut.heightList[0:self.ndatadec]
685 686
686 687 dataOut.flagDecodeData = True #asumo q la data no esta decodificada
687 688
688 689 if self.__profIndex == self.nCode-1:
689 690 self.__profIndex = 0
690 691 return 1
691 692
692 693 self.__profIndex += 1
693 694
694 695 return 1
695 696 # dataOut.flagDeflipData = True #asumo q la data no esta sin flip
696 697
697 698
698 699
699 700 class SpectraProc(ProcessingUnit):
700 701
701 702 def __init__(self):
702 703
703 704 self.objectDict = {}
704 705 self.buffer = None
705 706 self.firstdatatime = None
706 707 self.profIndex = 0
707 708 self.dataOut = Spectra()
708 709
709 710 def __updateObjFromInput(self):
710 711
711 712 self.dataOut.timeZone = self.dataIn.timeZone
712 713 self.dataOut.dstFlag = self.dataIn.dstFlag
713 714 self.dataOut.errorCount = self.dataIn.errorCount
714 715 self.dataOut.useLocalTime = self.dataIn.useLocalTime
715 716
716 717 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
717 718 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
718 719 self.dataOut.channelList = self.dataIn.channelList
719 720 self.dataOut.heightList = self.dataIn.heightList
720 721 self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
721 722 # self.dataOut.nHeights = self.dataIn.nHeights
722 723 # self.dataOut.nChannels = self.dataIn.nChannels
723 724 self.dataOut.nBaud = self.dataIn.nBaud
724 725 self.dataOut.nCode = self.dataIn.nCode
725 726 self.dataOut.code = self.dataIn.code
726 727 self.dataOut.nProfiles = self.dataOut.nFFTPoints
727 728 # self.dataOut.channelIndexList = self.dataIn.channelIndexList
728 729 self.dataOut.flagTimeBlock = self.dataIn.flagTimeBlock
729 730 self.dataOut.utctime = self.firstdatatime
730 731 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
731 732 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip
732 733 # self.dataOut.flagShiftFFT = self.dataIn.flagShiftFFT
733 734 self.dataOut.nCohInt = self.dataIn.nCohInt
734 735 self.dataOut.nIncohInt = 1
735 736 self.dataOut.ippSeconds = self.dataIn.ippSeconds
736 737 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
737 738
738 739 self.dataOut.timeInterval = self.dataIn.timeInterval*self.dataOut.nFFTPoints*self.dataOut.nIncohInt
739 740 self.dataOut.frequency = self.dataIn.frequency
740 741 self.dataOut.realtime = self.dataIn.realtime
741 742
742 743 def __getFft(self):
743 744 """
744 745 Convierte valores de Voltaje a Spectra
745 746
746 747 Affected:
747 748 self.dataOut.data_spc
748 749 self.dataOut.data_cspc
749 750 self.dataOut.data_dc
750 751 self.dataOut.heightList
751 752 self.profIndex
752 753 self.buffer
753 754 self.dataOut.flagNoData
754 755 """
755 756 fft_volt = numpy.fft.fft(self.buffer,n=self.dataOut.nFFTPoints,axis=1)
756 757 fft_volt = fft_volt.astype(numpy.dtype('complex'))
757 758 dc = fft_volt[:,0,:]
758 759
759 760 #calculo de self-spectra
760 761 fft_volt = numpy.fft.fftshift(fft_volt,axes=(1,))
761 762 spc = fft_volt * numpy.conjugate(fft_volt)
762 763 spc = spc.real
763 764
764 765 blocksize = 0
765 766 blocksize += dc.size
766 767 blocksize += spc.size
767 768
768 769 cspc = None
769 770 pairIndex = 0
770 771 if self.dataOut.pairsList != None:
771 772 #calculo de cross-spectra
772 773 cspc = numpy.zeros((self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
773 774 for pair in self.dataOut.pairsList:
774 775 cspc[pairIndex,:,:] = fft_volt[pair[0],:,:] * numpy.conjugate(fft_volt[pair[1],:,:])
775 776 pairIndex += 1
776 777 blocksize += cspc.size
777 778
778 779 self.dataOut.data_spc = spc
779 780 self.dataOut.data_cspc = cspc
780 781 self.dataOut.data_dc = dc
781 782 self.dataOut.blockSize = blocksize
782 783 self.dataOut.flagShiftFFT = False
783 784
784 785 def init(self, nProfiles=None, nFFTPoints=None, pairsList=None, ippFactor=None):
785 786
786 787 self.dataOut.flagNoData = True
787 788
788 789 if self.dataIn.type == "Spectra":
789 790 self.dataOut.copy(self.dataIn)
790 791 return
791 792
792 793 if self.dataIn.type == "Voltage":
793 794
794 795 if nFFTPoints == None:
795 796 raise ValueError, "This SpectraProc.init() need nFFTPoints input variable"
796 797
797 798 if pairsList == None:
798 799 nPairs = 0
799 800 else:
800 801 nPairs = len(pairsList)
801 802
802 803 if ippFactor == None:
803 804 ippFactor = 1
804 805 self.dataOut.ippFactor = ippFactor
805 806
806 807 self.dataOut.nFFTPoints = nFFTPoints
807 808 self.dataOut.pairsList = pairsList
808 809 self.dataOut.nPairs = nPairs
809 810
810 811 if self.buffer == None:
811 812 self.buffer = numpy.zeros((self.dataIn.nChannels,
812 813 nProfiles,
813 814 self.dataIn.nHeights),
814 815 dtype='complex')
815 816
816 817
817 818 self.buffer[:,self.profIndex,:] = self.dataIn.data.copy()
818 819 self.profIndex += 1
819 820
820 821 if self.firstdatatime == None:
821 822 self.firstdatatime = self.dataIn.utctime
822 823
823 824 if self.profIndex == nProfiles:
824 825 self.__updateObjFromInput()
825 826 self.__getFft()
826 827
827 828 self.dataOut.flagNoData = False
828 829
829 830 self.buffer = None
830 831 self.firstdatatime = None
831 832 self.profIndex = 0
832 833
833 834 return
834 835
835 836 raise ValueError, "The type object %s is not valid"%(self.dataIn.type)
836 837
837 838 def selectChannels(self, channelList):
838 839
839 840 channelIndexList = []
840 841
841 842 for channel in channelList:
842 843 index = self.dataOut.channelList.index(channel)
843 844 channelIndexList.append(index)
844 845
845 846 self.selectChannelsByIndex(channelIndexList)
846 847
847 848 def selectChannelsByIndex(self, channelIndexList):
848 849 """
849 850 Selecciona un bloque de datos en base a canales segun el channelIndexList
850 851
851 852 Input:
852 853 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
853 854
854 855 Affected:
855 856 self.dataOut.data_spc
856 857 self.dataOut.channelIndexList
857 858 self.dataOut.nChannels
858 859
859 860 Return:
860 861 None
861 862 """
862 863
863 864 for channelIndex in channelIndexList:
864 865 if channelIndex not in self.dataOut.channelIndexList:
865 866 print channelIndexList
866 867 raise ValueError, "The value %d in channelIndexList is not valid" %channelIndex
867 868
868 869 nChannels = len(channelIndexList)
869 870
870 871 data_spc = self.dataOut.data_spc[channelIndexList,:]
871 872
872 873 self.dataOut.data_spc = data_spc
873 874 self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
874 875 # self.dataOut.nChannels = nChannels
875 876
876 877 return 1
877 878
878 879 def selectHeights(self, minHei, maxHei):
879 880 """
880 881 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
881 882 minHei <= height <= maxHei
882 883
883 884 Input:
884 885 minHei : valor minimo de altura a considerar
885 886 maxHei : valor maximo de altura a considerar
886 887
887 888 Affected:
888 889 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
889 890
890 891 Return:
891 892 1 si el metodo se ejecuto con exito caso contrario devuelve 0
892 893 """
893 894 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
894 895 raise ValueError, "some value in (%d,%d) is not valid" % (minHei, maxHei)
895 896
896 897 if (maxHei > self.dataOut.heightList[-1]):
897 898 maxHei = self.dataOut.heightList[-1]
898 899 # raise ValueError, "some value in (%d,%d) is not valid" % (minHei, maxHei)
899 900
900 901 minIndex = 0
901 902 maxIndex = 0
902 903 heights = self.dataOut.heightList
903 904
904 905 inda = numpy.where(heights >= minHei)
905 906 indb = numpy.where(heights <= maxHei)
906 907
907 908 try:
908 909 minIndex = inda[0][0]
909 910 except:
910 911 minIndex = 0
911 912
912 913 try:
913 914 maxIndex = indb[0][-1]
914 915 except:
915 916 maxIndex = len(heights)
916 917
917 918 self.selectHeightsByIndex(minIndex, maxIndex)
918 919
919 920 return 1
920 921
921 922 def getBeaconSignal(self, tauindex = 0, channelindex = 0):
922 923 newheis = numpy.where(self.dataOut.heightList>self.dataOut.radarControllerHeaderObj.Taus[tauindex])
923 924 minIndex = min(newheis[0])
924 925 maxIndex = max(newheis[0])
925 926 data_spc = self.dataOut.data_spc[:,:,minIndex:maxIndex+1]
926 927 heightList = self.dataOut.heightList[minIndex:maxIndex+1]
927 928
928 929 # determina indices
929 930 nheis = int(self.dataOut.radarControllerHeaderObj.txB/(self.dataOut.heightList[1]-self.dataOut.heightList[0]))
930 931 avg_dB = 10*numpy.log10(numpy.sum(data_spc[channelindex,:,:],axis=0))
931 932 beacon_dB = numpy.sort(avg_dB)[-nheis:]
932 933 beacon_heiIndexList = []
933 934 for val in avg_dB.tolist():
934 935 if val >= beacon_dB[0]:
935 936 beacon_heiIndexList.append(avg_dB.tolist().index(val))
936 937
937 938 #data_spc = data_spc[:,:,beacon_heiIndexList]
938 939 data_cspc = None
939 940 if self.dataOut.data_cspc != None:
940 941 data_cspc = self.dataOut.data_cspc[:,:,minIndex:maxIndex+1]
941 942 #data_cspc = data_cspc[:,:,beacon_heiIndexList]
942 943
943 944 data_dc = None
944 945 if self.dataOut.data_dc != None:
945 946 data_dc = self.dataOut.data_dc[:,minIndex:maxIndex+1]
946 947 #data_dc = data_dc[:,beacon_heiIndexList]
947 948
948 949 self.dataOut.data_spc = data_spc
949 950 self.dataOut.data_cspc = data_cspc
950 951 self.dataOut.data_dc = data_dc
951 952 self.dataOut.heightList = heightList
952 953 self.dataOut.beacon_heiIndexList = beacon_heiIndexList
953 954
954 955 return 1
955 956
956 957
957 958 def selectHeightsByIndex(self, minIndex, maxIndex):
958 959 """
959 960 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
960 961 minIndex <= index <= maxIndex
961 962
962 963 Input:
963 964 minIndex : valor de indice minimo de altura a considerar
964 965 maxIndex : valor de indice maximo de altura a considerar
965 966
966 967 Affected:
967 968 self.dataOut.data_spc
968 969 self.dataOut.data_cspc
969 970 self.dataOut.data_dc
970 971 self.dataOut.heightList
971 972
972 973 Return:
973 974 1 si el metodo se ejecuto con exito caso contrario devuelve 0
974 975 """
975 976
976 977 if (minIndex < 0) or (minIndex > maxIndex):
977 978 raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
978 979
979 980 if (maxIndex >= self.dataOut.nHeights):
980 981 maxIndex = self.dataOut.nHeights-1
981 982 # raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
982 983
983 984 nHeights = maxIndex - minIndex + 1
984 985
985 986 #Spectra
986 987 data_spc = self.dataOut.data_spc[:,:,minIndex:maxIndex+1]
987 988
988 989 data_cspc = None
989 990 if self.dataOut.data_cspc != None:
990 991 data_cspc = self.dataOut.data_cspc[:,:,minIndex:maxIndex+1]
991 992
992 993 data_dc = None
993 994 if self.dataOut.data_dc != None:
994 995 data_dc = self.dataOut.data_dc[:,minIndex:maxIndex+1]
995 996
996 997 self.dataOut.data_spc = data_spc
997 998 self.dataOut.data_cspc = data_cspc
998 999 self.dataOut.data_dc = data_dc
999 1000
1000 1001 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex+1]
1001 1002
1002 1003 return 1
1003 1004
1004 1005 def removeDC(self, mode = 2):
1005 1006 jspectra = self.dataOut.data_spc
1006 1007 jcspectra = self.dataOut.data_cspc
1007 1008
1008 1009
1009 1010 num_chan = jspectra.shape[0]
1010 1011 num_hei = jspectra.shape[2]
1011 1012
1012 1013 if jcspectra != None:
1013 1014 jcspectraExist = True
1014 1015 num_pairs = jcspectra.shape[0]
1015 1016 else: jcspectraExist = False
1016 1017
1017 1018 freq_dc = jspectra.shape[1]/2
1018 1019 ind_vel = numpy.array([-2,-1,1,2]) + freq_dc
1019 1020
1020 1021 if ind_vel[0]<0:
1021 1022 ind_vel[range(0,1)] = ind_vel[range(0,1)] + self.num_prof
1022 1023
1023 1024 if mode == 1:
1024 1025 jspectra[:,freq_dc,:] = (jspectra[:,ind_vel[1],:] + jspectra[:,ind_vel[2],:])/2 #CORRECCION
1025 1026
1026 1027 if jcspectraExist:
1027 1028 jcspectra[:,freq_dc,:] = (jcspectra[:,ind_vel[1],:] + jcspectra[:,ind_vel[2],:])/2
1028 1029
1029 1030 if mode == 2:
1030 1031
1031 1032 vel = numpy.array([-2,-1,1,2])
1032 1033 xx = numpy.zeros([4,4])
1033 1034
1034 1035 for fil in range(4):
1035 1036 xx[fil,:] = vel[fil]**numpy.asarray(range(4))
1036 1037
1037 1038 xx_inv = numpy.linalg.inv(xx)
1038 1039 xx_aux = xx_inv[0,:]
1039 1040
1040 1041 for ich in range(num_chan):
1041 1042 yy = jspectra[ich,ind_vel,:]
1042 1043 jspectra[ich,freq_dc,:] = numpy.dot(xx_aux,yy)
1043 1044
1044 1045 junkid = jspectra[ich,freq_dc,:]<=0
1045 1046 cjunkid = sum(junkid)
1046 1047
1047 1048 if cjunkid.any():
1048 1049 jspectra[ich,freq_dc,junkid.nonzero()] = (jspectra[ich,ind_vel[1],junkid] + jspectra[ich,ind_vel[2],junkid])/2
1049 1050
1050 1051 if jcspectraExist:
1051 1052 for ip in range(num_pairs):
1052 1053 yy = jcspectra[ip,ind_vel,:]
1053 1054 jcspectra[ip,freq_dc,:] = numpy.dot(xx_aux,yy)
1054 1055
1055 1056
1056 1057 self.dataOut.data_spc = jspectra
1057 1058 self.dataOut.data_cspc = jcspectra
1058 1059
1059 1060 return 1
1060 1061
1061 1062 def removeInterference(self, interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None):
1062 1063
1063 1064 jspectra = self.dataOut.data_spc
1064 1065 jcspectra = self.dataOut.data_cspc
1065 1066 jnoise = self.dataOut.getNoise()
1066 1067 num_incoh = self.dataOut.nIncohInt
1067 1068
1068 1069 num_channel = jspectra.shape[0]
1069 1070 num_prof = jspectra.shape[1]
1070 1071 num_hei = jspectra.shape[2]
1071 1072
1072 1073 #hei_interf
1073 1074 if hei_interf == None:
1074 1075 count_hei = num_hei/2 #Como es entero no importa
1075 1076 hei_interf = numpy.asmatrix(range(count_hei)) + num_hei - count_hei
1076 1077 hei_interf = numpy.asarray(hei_interf)[0]
1077 1078 #nhei_interf
1078 1079 if (nhei_interf == None):
1079 1080 nhei_interf = 5
1080 1081 if (nhei_interf < 1):
1081 1082 nhei_interf = 1
1082 1083 if (nhei_interf > count_hei):
1083 1084 nhei_interf = count_hei
1084 1085 if (offhei_interf == None):
1085 1086 offhei_interf = 0
1086 1087
1087 1088 ind_hei = range(num_hei)
1088 1089 # mask_prof = numpy.asarray(range(num_prof - 2)) + 1
1089 1090 # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
1090 1091 mask_prof = numpy.asarray(range(num_prof))
1091 1092 num_mask_prof = mask_prof.size
1092 1093 comp_mask_prof = [0, num_prof/2]
1093 1094
1094 1095
1095 1096 #noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
1096 1097 if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
1097 1098 jnoise = numpy.nan
1098 1099 noise_exist = jnoise[0] < numpy.Inf
1099 1100
1100 1101 #Subrutina de Remocion de la Interferencia
1101 1102 for ich in range(num_channel):
1102 1103 #Se ordena los espectros segun su potencia (menor a mayor)
1103 1104 power = jspectra[ich,mask_prof,:]
1104 1105 power = power[:,hei_interf]
1105 1106 power = power.sum(axis = 0)
1106 1107 psort = power.ravel().argsort()
1107 1108
1108 1109 #Se estima la interferencia promedio en los Espectros de Potencia empleando
1109 1110 junkspc_interf = jspectra[ich,:,hei_interf[psort[range(offhei_interf, nhei_interf + offhei_interf)]]]
1110 1111
1111 1112 if noise_exist:
1112 1113 # tmp_noise = jnoise[ich] / num_prof
1113 1114 tmp_noise = jnoise[ich]
1114 1115 junkspc_interf = junkspc_interf - tmp_noise
1115 1116 #junkspc_interf[:,comp_mask_prof] = 0
1116 1117
1117 1118 jspc_interf = junkspc_interf.sum(axis = 0) / nhei_interf
1118 1119 jspc_interf = jspc_interf.transpose()
1119 1120 #Calculando el espectro de interferencia promedio
1120 1121 noiseid = numpy.where(jspc_interf <= tmp_noise/ math.sqrt(num_incoh))
1121 1122 noiseid = noiseid[0]
1122 1123 cnoiseid = noiseid.size
1123 1124 interfid = numpy.where(jspc_interf > tmp_noise/ math.sqrt(num_incoh))
1124 1125 interfid = interfid[0]
1125 1126 cinterfid = interfid.size
1126 1127
1127 1128 if (cnoiseid > 0): jspc_interf[noiseid] = 0
1128 1129
1129 1130 #Expandiendo los perfiles a limpiar
1130 1131 if (cinterfid > 0):
1131 1132 new_interfid = (numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof)%num_prof
1132 1133 new_interfid = numpy.asarray(new_interfid)
1133 1134 new_interfid = {x for x in new_interfid}
1134 1135 new_interfid = numpy.array(list(new_interfid))
1135 1136 new_cinterfid = new_interfid.size
1136 1137 else: new_cinterfid = 0
1137 1138
1138 1139 for ip in range(new_cinterfid):
1139 1140 ind = junkspc_interf[:,new_interfid[ip]].ravel().argsort()
1140 1141 jspc_interf[new_interfid[ip]] = junkspc_interf[ind[nhei_interf/2],new_interfid[ip]]
1141 1142
1142 1143
1143 1144 jspectra[ich,:,ind_hei] = jspectra[ich,:,ind_hei] - jspc_interf #Corregir indices
1144 1145
1145 1146 #Removiendo la interferencia del punto de mayor interferencia
1146 1147 ListAux = jspc_interf[mask_prof].tolist()
1147 1148 maxid = ListAux.index(max(ListAux))
1148 1149
1149 1150
1150 1151 if cinterfid > 0:
1151 1152 for ip in range(cinterfid*(interf == 2) - 1):
1152 1153 ind = (jspectra[ich,interfid[ip],:] < tmp_noise*(1 + 1/math.sqrt(num_incoh))).nonzero()
1153 1154 cind = len(ind)
1154 1155
1155 1156 if (cind > 0):
1156 1157 jspectra[ich,interfid[ip],ind] = tmp_noise*(1 + (numpy.random.uniform(cind) - 0.5)/math.sqrt(num_incoh))
1157 1158
1158 1159 ind = numpy.array([-2,-1,1,2])
1159 1160 xx = numpy.zeros([4,4])
1160 1161
1161 1162 for id1 in range(4):
1162 1163 xx[:,id1] = ind[id1]**numpy.asarray(range(4))
1163 1164
1164 1165 xx_inv = numpy.linalg.inv(xx)
1165 1166 xx = xx_inv[:,0]
1166 1167 ind = (ind + maxid + num_mask_prof)%num_mask_prof
1167 1168 yy = jspectra[ich,mask_prof[ind],:]
1168 1169 jspectra[ich,mask_prof[maxid],:] = numpy.dot(yy.transpose(),xx)
1169 1170
1170 1171
1171 1172 indAux = (jspectra[ich,:,:] < tmp_noise*(1-1/math.sqrt(num_incoh))).nonzero()
1172 1173 jspectra[ich,indAux[0],indAux[1]] = tmp_noise * (1 - 1/math.sqrt(num_incoh))
1173 1174
1174 1175 #Remocion de Interferencia en el Cross Spectra
1175 1176 if jcspectra == None: return jspectra, jcspectra
1176 1177 num_pairs = jcspectra.size/(num_prof*num_hei)
1177 1178 jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
1178 1179
1179 1180 for ip in range(num_pairs):
1180 1181
1181 1182 #-------------------------------------------
1182 1183
1183 1184 cspower = numpy.abs(jcspectra[ip,mask_prof,:])
1184 1185 cspower = cspower[:,hei_interf]
1185 1186 cspower = cspower.sum(axis = 0)
1186 1187
1187 1188 cspsort = cspower.ravel().argsort()
1188 1189 junkcspc_interf = jcspectra[ip,:,hei_interf[cspsort[range(offhei_interf, nhei_interf + offhei_interf)]]]
1189 1190 junkcspc_interf = junkcspc_interf.transpose()
1190 1191 jcspc_interf = junkcspc_interf.sum(axis = 1)/nhei_interf
1191 1192
1192 1193 ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
1193 1194
1194 1195 median_real = numpy.median(numpy.real(junkcspc_interf[mask_prof[ind[range(3*num_prof/4)]],:]))
1195 1196 median_imag = numpy.median(numpy.imag(junkcspc_interf[mask_prof[ind[range(3*num_prof/4)]],:]))
1196 1197 junkcspc_interf[comp_mask_prof,:] = numpy.complex(median_real, median_imag)
1197 1198
1198 1199 for iprof in range(num_prof):
1199 1200 ind = numpy.abs(junkcspc_interf[iprof,:]).ravel().argsort()
1200 1201 jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf/2]]
1201 1202
1202 1203 #Removiendo la Interferencia
1203 1204 jcspectra[ip,:,ind_hei] = jcspectra[ip,:,ind_hei] - jcspc_interf
1204 1205
1205 1206 ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
1206 1207 maxid = ListAux.index(max(ListAux))
1207 1208
1208 1209 ind = numpy.array([-2,-1,1,2])
1209 1210 xx = numpy.zeros([4,4])
1210 1211
1211 1212 for id1 in range(4):
1212 1213 xx[:,id1] = ind[id1]**numpy.asarray(range(4))
1213 1214
1214 1215 xx_inv = numpy.linalg.inv(xx)
1215 1216 xx = xx_inv[:,0]
1216 1217
1217 1218 ind = (ind + maxid + num_mask_prof)%num_mask_prof
1218 1219 yy = jcspectra[ip,mask_prof[ind],:]
1219 1220 jcspectra[ip,mask_prof[maxid],:] = numpy.dot(yy.transpose(),xx)
1220 1221
1221 1222 #Guardar Resultados
1222 1223 self.dataOut.data_spc = jspectra
1223 1224 self.dataOut.data_cspc = jcspectra
1224 1225
1225 1226 return 1
1226 1227
1227 1228 def setRadarFrequency(self, frequency=None):
1228 1229 if frequency != None:
1229 1230 self.dataOut.frequency = frequency
1230 1231
1231 1232 return 1
1232 1233
1233 1234 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
1234 1235 #validacion de rango
1235 1236 if minHei == None:
1236 1237 minHei = self.dataOut.heightList[0]
1237 1238
1238 1239 if maxHei == None:
1239 1240 maxHei = self.dataOut.heightList[-1]
1240 1241
1241 1242 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
1242 1243 print 'minHei: %.2f is out of the heights range'%(minHei)
1243 1244 print 'minHei is setting to %.2f'%(self.dataOut.heightList[0])
1244 1245 minHei = self.dataOut.heightList[0]
1245 1246
1246 1247 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
1247 1248 print 'maxHei: %.2f is out of the heights range'%(maxHei)
1248 1249 print 'maxHei is setting to %.2f'%(self.dataOut.heightList[-1])
1249 1250 maxHei = self.dataOut.heightList[-1]
1250 1251
1251 1252 # validacion de velocidades
1252 1253 velrange = self.dataOut.getVelRange(1)
1253 1254
1254 1255 if minVel == None:
1255 1256 minVel = velrange[0]
1256 1257
1257 1258 if maxVel == None:
1258 1259 maxVel = velrange[-1]
1259 1260
1260 1261 if (minVel < velrange[0]) or (minVel > maxVel):
1261 1262 print 'minVel: %.2f is out of the velocity range'%(minVel)
1262 1263 print 'minVel is setting to %.2f'%(velrange[0])
1263 1264 minVel = velrange[0]
1264 1265
1265 1266 if (maxVel > velrange[-1]) or (maxVel < minVel):
1266 1267 print 'maxVel: %.2f is out of the velocity range'%(maxVel)
1267 1268 print 'maxVel is setting to %.2f'%(velrange[-1])
1268 1269 maxVel = velrange[-1]
1269 1270
1270 1271 # seleccion de indices para rango
1271 1272 minIndex = 0
1272 1273 maxIndex = 0
1273 1274 heights = self.dataOut.heightList
1274 1275
1275 1276 inda = numpy.where(heights >= minHei)
1276 1277 indb = numpy.where(heights <= maxHei)
1277 1278
1278 1279 try:
1279 1280 minIndex = inda[0][0]
1280 1281 except:
1281 1282 minIndex = 0
1282 1283
1283 1284 try:
1284 1285 maxIndex = indb[0][-1]
1285 1286 except:
1286 1287 maxIndex = len(heights)
1287 1288
1288 1289 if (minIndex < 0) or (minIndex > maxIndex):
1289 1290 raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
1290 1291
1291 1292 if (maxIndex >= self.dataOut.nHeights):
1292 1293 maxIndex = self.dataOut.nHeights-1
1293 1294
1294 1295 # seleccion de indices para velocidades
1295 1296 indminvel = numpy.where(velrange >= minVel)
1296 1297 indmaxvel = numpy.where(velrange <= maxVel)
1297 1298 try:
1298 1299 minIndexVel = indminvel[0][0]
1299 1300 except:
1300 1301 minIndexVel = 0
1301 1302
1302 1303 try:
1303 1304 maxIndexVel = indmaxvel[0][-1]
1304 1305 except:
1305 1306 maxIndexVel = len(velrange)
1306 1307
1307 1308 #seleccion del espectro
1308 1309 data_spc = self.dataOut.data_spc[:,minIndexVel:maxIndexVel+1,minIndex:maxIndex+1]
1309 1310 #estimacion de ruido
1310 1311 noise = numpy.zeros(self.dataOut.nChannels)
1311 1312
1312 1313 for channel in range(self.dataOut.nChannels):
1313 1314 daux = data_spc[channel,:,:]
1314 1315 noise[channel] = hildebrand_sekhon(daux, self.dataOut.nIncohInt)
1315 1316
1316 1317 self.dataOut.noise = noise.copy()
1317 1318
1318 1319 return 1
1319 1320
1320 1321
1321 1322 class IncohInt(Operation):
1322 1323
1323 1324
1324 1325 __profIndex = 0
1325 1326 __withOverapping = False
1326 1327
1327 1328 __byTime = False
1328 1329 __initime = None
1329 1330 __lastdatatime = None
1330 1331 __integrationtime = None
1331 1332
1332 1333 __buffer_spc = None
1333 1334 __buffer_cspc = None
1334 1335 __buffer_dc = None
1335 1336
1336 1337 __dataReady = False
1337 1338
1338 1339 __timeInterval = None
1339 1340
1340 1341 n = None
1341 1342
1342 1343
1343 1344
1344 1345 def __init__(self):
1345 1346
1346 1347 self.__isConfig = False
1347 1348
1348 1349 def setup(self, n=None, timeInterval=None, overlapping=False):
1349 1350 """
1350 1351 Set the parameters of the integration class.
1351 1352
1352 1353 Inputs:
1353 1354
1354 1355 n : Number of coherent integrations
1355 1356 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1356 1357 overlapping :
1357 1358
1358 1359 """
1359 1360
1360 1361 self.__initime = None
1361 1362 self.__lastdatatime = 0
1362 1363 self.__buffer_spc = None
1363 1364 self.__buffer_cspc = None
1364 1365 self.__buffer_dc = None
1365 1366 self.__dataReady = False
1366 1367
1367 1368
1368 1369 if n == None and timeInterval == None:
1369 1370 raise ValueError, "n or timeInterval should be specified ..."
1370 1371
1371 1372 if n != None:
1372 1373 self.n = n
1373 1374 self.__byTime = False
1374 1375 else:
1375 1376 self.__integrationtime = timeInterval #if (type(timeInterval)!=integer) -> change this line
1376 1377 self.n = 9999
1377 1378 self.__byTime = True
1378 1379
1379 1380 if overlapping:
1380 1381 self.__withOverapping = True
1381 1382 else:
1382 1383 self.__withOverapping = False
1383 1384 self.__buffer_spc = 0
1384 1385 self.__buffer_cspc = 0
1385 1386 self.__buffer_dc = 0
1386 1387
1387 1388 self.__profIndex = 0
1388 1389
1389 1390 def putData(self, data_spc, data_cspc, data_dc):
1390 1391
1391 1392 """
1392 1393 Add a profile to the __buffer_spc and increase in one the __profileIndex
1393 1394
1394 1395 """
1395 1396
1396 1397 if not self.__withOverapping:
1397 1398 self.__buffer_spc += data_spc
1398 1399
1399 1400 if data_cspc == None:
1400 1401 self.__buffer_cspc = None
1401 1402 else:
1402 1403 self.__buffer_cspc += data_cspc
1403 1404
1404 1405 if data_dc == None:
1405 1406 self.__buffer_dc = None
1406 1407 else:
1407 1408 self.__buffer_dc += data_dc
1408 1409
1409 1410 self.__profIndex += 1
1410 1411 return
1411 1412
1412 1413 #Overlapping data
1413 1414 nChannels, nFFTPoints, nHeis = data_spc.shape
1414 1415 data_spc = numpy.reshape(data_spc, (1, nChannels, nFFTPoints, nHeis))
1415 1416 if data_cspc != None:
1416 1417 data_cspc = numpy.reshape(data_cspc, (1, -1, nFFTPoints, nHeis))
1417 1418 if data_dc != None:
1418 1419 data_dc = numpy.reshape(data_dc, (1, -1, nHeis))
1419 1420
1420 1421 #If the buffer is empty then it takes the data value
1421 1422 if self.__buffer_spc == None:
1422 1423 self.__buffer_spc = data_spc
1423 1424
1424 1425 if data_cspc == None:
1425 1426 self.__buffer_cspc = None
1426 1427 else:
1427 1428 self.__buffer_cspc += data_cspc
1428 1429
1429 1430 if data_dc == None:
1430 1431 self.__buffer_dc = None
1431 1432 else:
1432 1433 self.__buffer_dc += data_dc
1433 1434
1434 1435 self.__profIndex += 1
1435 1436 return
1436 1437
1437 1438 #If the buffer length is lower than n then stakcing the data value
1438 1439 if self.__profIndex < self.n:
1439 1440 self.__buffer_spc = numpy.vstack((self.__buffer_spc, data_spc))
1440 1441
1441 1442 if data_cspc != None:
1442 1443 self.__buffer_cspc = numpy.vstack((self.__buffer_cspc, data_cspc))
1443 1444
1444 1445 if data_dc != None:
1445 1446 self.__buffer_dc = numpy.vstack((self.__buffer_dc, data_dc))
1446 1447
1447 1448 self.__profIndex += 1
1448 1449 return
1449 1450
1450 1451 #If the buffer length is equal to n then replacing the last buffer value with the data value
1451 1452 self.__buffer_spc = numpy.roll(self.__buffer_spc, -1, axis=0)
1452 1453 self.__buffer_spc[self.n-1] = data_spc
1453 1454
1454 1455 if data_cspc != None:
1455 1456 self.__buffer_cspc = numpy.roll(self.__buffer_cspc, -1, axis=0)
1456 1457 self.__buffer_cspc[self.n-1] = data_cspc
1457 1458
1458 1459 if data_dc != None:
1459 1460 self.__buffer_dc = numpy.roll(self.__buffer_dc, -1, axis=0)
1460 1461 self.__buffer_dc[self.n-1] = data_dc
1461 1462
1462 1463 self.__profIndex = self.n
1463 1464 return
1464 1465
1465 1466
1466 1467 def pushData(self):
1467 1468 """
1468 1469 Return the sum of the last profiles and the profiles used in the sum.
1469 1470
1470 1471 Affected:
1471 1472
1472 1473 self.__profileIndex
1473 1474
1474 1475 """
1475 1476 data_spc = None
1476 1477 data_cspc = None
1477 1478 data_dc = None
1478 1479
1479 1480 if not self.__withOverapping:
1480 1481 data_spc = self.__buffer_spc
1481 1482 data_cspc = self.__buffer_cspc
1482 1483 data_dc = self.__buffer_dc
1483 1484
1484 1485 n = self.__profIndex
1485 1486
1486 1487 self.__buffer_spc = 0
1487 1488 self.__buffer_cspc = 0
1488 1489 self.__buffer_dc = 0
1489 1490 self.__profIndex = 0
1490 1491
1491 1492 return data_spc, data_cspc, data_dc, n
1492 1493
1493 1494 #Integration with Overlapping
1494 1495 data_spc = numpy.sum(self.__buffer_spc, axis=0)
1495 1496
1496 1497 if self.__buffer_cspc != None:
1497 1498 data_cspc = numpy.sum(self.__buffer_cspc, axis=0)
1498 1499
1499 1500 if self.__buffer_dc != None:
1500 1501 data_dc = numpy.sum(self.__buffer_dc, axis=0)
1501 1502
1502 1503 n = self.__profIndex
1503 1504
1504 1505 return data_spc, data_cspc, data_dc, n
1505 1506
1506 1507 def byProfiles(self, *args):
1507 1508
1508 1509 self.__dataReady = False
1509 1510 avgdata_spc = None
1510 1511 avgdata_cspc = None
1511 1512 avgdata_dc = None
1512 1513 n = None
1513 1514
1514 1515 self.putData(*args)
1515 1516
1516 1517 if self.__profIndex == self.n:
1517 1518
1518 1519 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1519 1520 self.__dataReady = True
1520 1521
1521 1522 return avgdata_spc, avgdata_cspc, avgdata_dc
1522 1523
1523 1524 def byTime(self, datatime, *args):
1524 1525
1525 1526 self.__dataReady = False
1526 1527 avgdata_spc = None
1527 1528 avgdata_cspc = None
1528 1529 avgdata_dc = None
1529 1530 n = None
1530 1531
1531 1532 self.putData(*args)
1532 1533
1533 1534 if (datatime - self.__initime) >= self.__integrationtime:
1534 1535 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1535 1536 self.n = n
1536 1537 self.__dataReady = True
1537 1538
1538 1539 return avgdata_spc, avgdata_cspc, avgdata_dc
1539 1540
1540 1541 def integrate(self, datatime, *args):
1541 1542
1542 1543 if self.__initime == None:
1543 1544 self.__initime = datatime
1544 1545
1545 1546 if self.__byTime:
1546 1547 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(datatime, *args)
1547 1548 else:
1548 1549 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1549 1550
1550 1551 self.__lastdatatime = datatime
1551 1552
1552 1553 if avgdata_spc == None:
1553 1554 return None, None, None, None
1554 1555
1555 1556 avgdatatime = self.__initime
1556 1557 try:
1557 1558 self.__timeInterval = (self.__lastdatatime - self.__initime)/(self.n - 1)
1558 1559 except:
1559 1560 self.__timeInterval = self.__lastdatatime - self.__initime
1560 1561
1561 1562 deltatime = datatime -self.__lastdatatime
1562 1563
1563 1564 if not self.__withOverapping:
1564 1565 self.__initime = datatime
1565 1566 else:
1566 1567 self.__initime += deltatime
1567 1568
1568 1569 return avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc
1569 1570
1570 1571 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
1571 1572
1572 1573 if n==1:
1573 1574 dataOut.flagNoData = False
1574 1575 return
1575 1576
1576 1577 if not self.__isConfig:
1577 1578 self.setup(n, timeInterval, overlapping)
1578 1579 self.__isConfig = True
1579 1580
1580 1581 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
1581 1582 dataOut.data_spc,
1582 1583 dataOut.data_cspc,
1583 1584 dataOut.data_dc)
1584 1585
1585 1586 # dataOut.timeInterval *= n
1586 1587 dataOut.flagNoData = True
1587 1588
1588 1589 if self.__dataReady:
1589 1590
1590 1591 dataOut.data_spc = avgdata_spc
1591 1592 dataOut.data_cspc = avgdata_cspc
1592 1593 dataOut.data_dc = avgdata_dc
1593 1594
1594 1595 dataOut.nIncohInt *= self.n
1595 1596 dataOut.utctime = avgdatatime
1596 1597 #dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt * dataOut.nIncohInt * dataOut.nFFTPoints
1597 1598 dataOut.timeInterval = self.__timeInterval*self.n
1598 1599 dataOut.flagNoData = False
1599 1600
1600 1601 class ProfileConcat(Operation):
1601 1602
1602 1603 __isConfig = False
1603 1604 buffer = None
1604 1605
1605 1606 def __init__(self):
1606 1607
1607 1608 self.profileIndex = 0
1608 1609
1609 1610 def reset(self):
1610 1611 self.buffer = numpy.zeros_like(self.buffer)
1611 1612 self.start_index = 0
1612 1613 self.times = 1
1613 1614
1614 1615 def setup(self, data, m, n=1):
1615 1616 self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
1616 1617 self.profiles = data.shape[1]
1617 1618 self.start_index = 0
1618 1619 self.times = 1
1619 1620
1620 1621 def concat(self, data):
1621 1622
1622 1623 self.buffer[:,self.start_index:self.profiles*self.times] = data.copy()
1623 1624 self.start_index = self.start_index + self.profiles
1624 1625
1625 1626 def run(self, dataOut, m):
1626 1627
1627 1628 dataOut.flagNoData = True
1628 1629
1629 1630 if not self.__isConfig:
1630 1631 self.setup(dataOut.data, m, 1)
1631 1632 self.__isConfig = True
1632 1633
1633 1634 self.concat(dataOut.data)
1634 1635 self.times += 1
1635 1636 if self.times > m:
1636 1637 dataOut.data = self.buffer
1637 1638 self.reset()
1638 1639 dataOut.flagNoData = False
1639 1640 # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
1640 1641 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1641 1642 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * 5
1642 1643 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
1643 1644
1644 1645
1645 1646
1646 1647 class ProfileSelector(Operation):
1647 1648
1648 1649 profileIndex = None
1649 1650 # Tamanho total de los perfiles
1650 1651 nProfiles = None
1651 1652
1652 1653 def __init__(self):
1653 1654
1654 1655 self.profileIndex = 0
1655 1656
1656 1657 def incIndex(self):
1657 1658 self.profileIndex += 1
1658 1659
1659 1660 if self.profileIndex >= self.nProfiles:
1660 1661 self.profileIndex = 0
1661 1662
1662 1663 def isProfileInRange(self, minIndex, maxIndex):
1663 1664
1664 1665 if self.profileIndex < minIndex:
1665 1666 return False
1666 1667
1667 1668 if self.profileIndex > maxIndex:
1668 1669 return False
1669 1670
1670 1671 return True
1671 1672
1672 1673 def isProfileInList(self, profileList):
1673 1674
1674 1675 if self.profileIndex not in profileList:
1675 1676 return False
1676 1677
1677 1678 return True
1678 1679
1679 1680 def run(self, dataOut, profileList=None, profileRangeList=None):
1680 1681
1681 1682 dataOut.flagNoData = True
1682 1683 self.nProfiles = dataOut.nProfiles
1683 1684
1684 1685 if profileList != None:
1685 1686 if self.isProfileInList(profileList):
1686 1687 dataOut.flagNoData = False
1687 1688
1688 1689 self.incIndex()
1689 1690 return 1
1690 1691
1691 1692
1692 1693 elif profileRangeList != None:
1693 1694 minIndex = profileRangeList[0]
1694 1695 maxIndex = profileRangeList[1]
1695 1696 if self.isProfileInRange(minIndex, maxIndex):
1696 1697 dataOut.flagNoData = False
1697 1698
1698 1699 self.incIndex()
1699 1700 return 1
1700 1701
1701 1702 else:
1702 1703 raise ValueError, "ProfileSelector needs profileList or profileRangeList"
1703 1704
1704 1705 return 0
1705 1706
1706 1707 class SpectraHeisProc(ProcessingUnit):
1707 1708 def __init__(self):
1708 1709 self.objectDict = {}
1709 1710 # self.buffer = None
1710 1711 # self.firstdatatime = None
1711 1712 # self.profIndex = 0
1712 1713 self.dataOut = SpectraHeis()
1713 1714
1714 1715 def __updateObjFromInput(self):
1715 1716 self.dataOut.timeZone = self.dataIn.timeZone
1716 1717 self.dataOut.dstFlag = self.dataIn.dstFlag
1717 1718 self.dataOut.errorCount = self.dataIn.errorCount
1718 1719 self.dataOut.useLocalTime = self.dataIn.useLocalTime
1719 1720
1720 1721 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()#
1721 1722 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()#
1722 1723 self.dataOut.channelList = self.dataIn.channelList
1723 1724 self.dataOut.heightList = self.dataIn.heightList
1724 1725 # self.dataOut.dtype = self.dataIn.dtype
1725 1726 self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
1726 1727 # self.dataOut.nHeights = self.dataIn.nHeights
1727 1728 # self.dataOut.nChannels = self.dataIn.nChannels
1728 1729 self.dataOut.nBaud = self.dataIn.nBaud
1729 1730 self.dataOut.nCode = self.dataIn.nCode
1730 1731 self.dataOut.code = self.dataIn.code
1731 1732 # self.dataOut.nProfiles = 1
1732 1733 # self.dataOut.nProfiles = self.dataOut.nFFTPoints
1733 1734 self.dataOut.nFFTPoints = self.dataIn.nHeights
1734 1735 # self.dataOut.channelIndexList = self.dataIn.channelIndexList
1735 1736 # self.dataOut.flagNoData = self.dataIn.flagNoData
1736 1737 self.dataOut.flagTimeBlock = self.dataIn.flagTimeBlock
1737 1738 self.dataOut.utctime = self.dataIn.utctime
1738 1739 # self.dataOut.utctime = self.firstdatatime
1739 1740 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
1740 1741 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip
1741 1742 # self.dataOut.flagShiftFFT = self.dataIn.flagShiftFFT
1742 1743 self.dataOut.nCohInt = self.dataIn.nCohInt
1743 1744 self.dataOut.nIncohInt = 1
1744 1745 self.dataOut.ippSeconds= self.dataIn.ippSeconds
1745 1746 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
1746 1747
1747 1748 self.dataOut.timeInterval = self.dataIn.timeInterval*self.dataOut.nIncohInt
1748 1749 # self.dataOut.set=self.dataIn.set
1749 1750 # self.dataOut.deltaHeight=self.dataIn.deltaHeight
1750 1751
1751 1752
1752 1753 def __updateObjFromFits(self):
1753 1754 self.dataOut.utctime = self.dataIn.utctime
1754 1755 self.dataOut.channelIndexList = self.dataIn.channelIndexList
1755 1756
1756 1757 self.dataOut.channelList = self.dataIn.channelList
1757 1758 self.dataOut.heightList = self.dataIn.heightList
1758 1759 self.dataOut.data_spc = self.dataIn.data
1759 1760 self.dataOut.timeInterval = self.dataIn.timeInterval
1760 1761 self.dataOut.timeZone = self.dataIn.timeZone
1761 1762 self.dataOut.useLocalTime = True
1762 1763 # self.dataOut.
1763 1764 # self.dataOut.
1764 1765
1765 1766 def __getFft(self):
1766 1767
1767 1768 fft_volt = numpy.fft.fft(self.dataIn.data, axis=1)
1768 1769 fft_volt = numpy.fft.fftshift(fft_volt,axes=(1,))
1769 1770 spc = numpy.abs(fft_volt * numpy.conjugate(fft_volt))/(self.dataOut.nFFTPoints)
1770 1771 self.dataOut.data_spc = spc
1771 1772
1772 1773 def init(self):
1773 1774
1774 1775 self.dataOut.flagNoData = True
1775 1776
1776 1777 if self.dataIn.type == "Fits":
1777 1778 self.__updateObjFromFits()
1778 1779 self.dataOut.flagNoData = False
1779 1780 return
1780 1781
1781 1782 if self.dataIn.type == "SpectraHeis":
1782 1783 self.dataOut.copy(self.dataIn)
1783 1784 return
1784 1785
1785 1786 if self.dataIn.type == "Voltage":
1786 1787 self.__updateObjFromInput()
1787 1788 self.__getFft()
1788 1789 self.dataOut.flagNoData = False
1789 1790
1790 1791 return
1791 1792
1792 1793 raise ValueError, "The type object %s is not valid"%(self.dataIn.type)
1793 1794
1794 1795
1795 1796 def selectChannels(self, channelList):
1796 1797
1797 1798 channelIndexList = []
1798 1799
1799 1800 for channel in channelList:
1800 1801 index = self.dataOut.channelList.index(channel)
1801 1802 channelIndexList.append(index)
1802 1803
1803 1804 self.selectChannelsByIndex(channelIndexList)
1804 1805
1805 1806 def selectChannelsByIndex(self, channelIndexList):
1806 1807 """
1807 1808 Selecciona un bloque de datos en base a canales segun el channelIndexList
1808 1809
1809 1810 Input:
1810 1811 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
1811 1812
1812 1813 Affected:
1813 1814 self.dataOut.data
1814 1815 self.dataOut.channelIndexList
1815 1816 self.dataOut.nChannels
1816 1817 self.dataOut.m_ProcessingHeader.totalSpectra
1817 1818 self.dataOut.systemHeaderObj.numChannels
1818 1819 self.dataOut.m_ProcessingHeader.blockSize
1819 1820
1820 1821 Return:
1821 1822 None
1822 1823 """
1823 1824
1824 1825 for channelIndex in channelIndexList:
1825 1826 if channelIndex not in self.dataOut.channelIndexList:
1826 1827 print channelIndexList
1827 1828 raise ValueError, "The value %d in channelIndexList is not valid" %channelIndex
1828 1829
1829 1830 nChannels = len(channelIndexList)
1830 1831
1831 1832 data_spc = self.dataOut.data_spc[channelIndexList,:]
1832 1833
1833 1834 self.dataOut.data_spc = data_spc
1834 1835 self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
1835 1836
1836 1837 return 1
1837 1838
1838 1839 class IncohInt4SpectraHeis(Operation):
1839 1840
1840 1841 __isConfig = False
1841 1842
1842 1843 __profIndex = 0
1843 1844 __withOverapping = False
1844 1845
1845 1846 __byTime = False
1846 1847 __initime = None
1847 1848 __lastdatatime = None
1848 1849 __integrationtime = None
1849 1850
1850 1851 __buffer = None
1851 1852
1852 1853 __dataReady = False
1853 1854
1854 1855 n = None
1855 1856
1856 1857
1857 1858 def __init__(self):
1858 1859
1859 1860 self.__isConfig = False
1860 1861
1861 1862 def setup(self, n=None, timeInterval=None, overlapping=False):
1862 1863 """
1863 1864 Set the parameters of the integration class.
1864 1865
1865 1866 Inputs:
1866 1867
1867 1868 n : Number of coherent integrations
1868 1869 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1869 1870 overlapping :
1870 1871
1871 1872 """
1872 1873
1873 1874 self.__initime = None
1874 1875 self.__lastdatatime = 0
1875 1876 self.__buffer = None
1876 1877 self.__dataReady = False
1877 1878
1878 1879
1879 1880 if n == None and timeInterval == None:
1880 1881 raise ValueError, "n or timeInterval should be specified ..."
1881 1882
1882 1883 if n != None:
1883 1884 self.n = n
1884 1885 self.__byTime = False
1885 1886 else:
1886 1887 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
1887 1888 self.n = 9999
1888 1889 self.__byTime = True
1889 1890
1890 1891 if overlapping:
1891 1892 self.__withOverapping = True
1892 1893 self.__buffer = None
1893 1894 else:
1894 1895 self.__withOverapping = False
1895 1896 self.__buffer = 0
1896 1897
1897 1898 self.__profIndex = 0
1898 1899
1899 1900 def putData(self, data):
1900 1901
1901 1902 """
1902 1903 Add a profile to the __buffer and increase in one the __profileIndex
1903 1904
1904 1905 """
1905 1906
1906 1907 if not self.__withOverapping:
1907 1908 self.__buffer += data.copy()
1908 1909 self.__profIndex += 1
1909 1910 return
1910 1911
1911 1912 #Overlapping data
1912 1913 nChannels, nHeis = data.shape
1913 1914 data = numpy.reshape(data, (1, nChannels, nHeis))
1914 1915
1915 1916 #If the buffer is empty then it takes the data value
1916 1917 if self.__buffer == None:
1917 1918 self.__buffer = data
1918 1919 self.__profIndex += 1
1919 1920 return
1920 1921
1921 1922 #If the buffer length is lower than n then stakcing the data value
1922 1923 if self.__profIndex < self.n:
1923 1924 self.__buffer = numpy.vstack((self.__buffer, data))
1924 1925 self.__profIndex += 1
1925 1926 return
1926 1927
1927 1928 #If the buffer length is equal to n then replacing the last buffer value with the data value
1928 1929 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
1929 1930 self.__buffer[self.n-1] = data
1930 1931 self.__profIndex = self.n
1931 1932 return
1932 1933
1933 1934
1934 1935 def pushData(self):
1935 1936 """
1936 1937 Return the sum of the last profiles and the profiles used in the sum.
1937 1938
1938 1939 Affected:
1939 1940
1940 1941 self.__profileIndex
1941 1942
1942 1943 """
1943 1944
1944 1945 if not self.__withOverapping:
1945 1946 data = self.__buffer
1946 1947 n = self.__profIndex
1947 1948
1948 1949 self.__buffer = 0
1949 1950 self.__profIndex = 0
1950 1951
1951 1952 return data, n
1952 1953
1953 1954 #Integration with Overlapping
1954 1955 data = numpy.sum(self.__buffer, axis=0)
1955 1956 n = self.__profIndex
1956 1957
1957 1958 return data, n
1958 1959
1959 1960 def byProfiles(self, data):
1960 1961
1961 1962 self.__dataReady = False
1962 1963 avgdata = None
1963 1964 n = None
1964 1965
1965 1966 self.putData(data)
1966 1967
1967 1968 if self.__profIndex == self.n:
1968 1969
1969 1970 avgdata, n = self.pushData()
1970 1971 self.__dataReady = True
1971 1972
1972 1973 return avgdata
1973 1974
1974 1975 def byTime(self, data, datatime):
1975 1976
1976 1977 self.__dataReady = False
1977 1978 avgdata = None
1978 1979 n = None
1979 1980
1980 1981 self.putData(data)
1981 1982
1982 1983 if (datatime - self.__initime) >= self.__integrationtime:
1983 1984 avgdata, n = self.pushData()
1984 1985 self.n = n
1985 1986 self.__dataReady = True
1986 1987
1987 1988 return avgdata
1988 1989
1989 1990 def integrate(self, data, datatime=None):
1990 1991
1991 1992 if self.__initime == None:
1992 1993 self.__initime = datatime
1993 1994
1994 1995 if self.__byTime:
1995 1996 avgdata = self.byTime(data, datatime)
1996 1997 else:
1997 1998 avgdata = self.byProfiles(data)
1998 1999
1999 2000
2000 2001 self.__lastdatatime = datatime
2001 2002
2002 2003 if avgdata == None:
2003 2004 return None, None
2004 2005
2005 2006 avgdatatime = self.__initime
2006 2007
2007 2008 deltatime = datatime -self.__lastdatatime
2008 2009
2009 2010 if not self.__withOverapping:
2010 2011 self.__initime = datatime
2011 2012 else:
2012 2013 self.__initime += deltatime
2013 2014
2014 2015 return avgdata, avgdatatime
2015 2016
2016 2017 def run(self, dataOut, **kwargs):
2017 2018
2018 2019 if not self.__isConfig:
2019 2020 self.setup(**kwargs)
2020 2021 self.__isConfig = True
2021 2022
2022 2023 avgdata, avgdatatime = self.integrate(dataOut.data_spc, dataOut.utctime)
2023 2024
2024 2025 # dataOut.timeInterval *= n
2025 2026 dataOut.flagNoData = True
2026 2027
2027 2028 if self.__dataReady:
2028 2029 dataOut.data_spc = avgdata
2029 2030 dataOut.nIncohInt *= self.n
2030 2031 # dataOut.nCohInt *= self.n
2031 2032 dataOut.utctime = avgdatatime
2032 2033 dataOut.timeInterval = dataOut.ippSeconds * dataOut.nIncohInt
2033 2034 # dataOut.timeInterval = self.__timeInterval*self.n
2034 2035 dataOut.flagNoData = False
2035 2036
2036 2037
2037 2038
2038 2039
2039 2040 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now