##// END OF EJS Templates
schain-jc
RSS
jroprocessing:...
Miguel Valdez -
r191:58293a972ca2
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,2471 +1,2473
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $
5 5 '''
6 6
7 7 import os, sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13
14 14 from jrodata import *
15 15 from jroheaderIO import *
16 16
17 17 def isNumber(str):
18 18 """
19 19 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
20 20
21 21 Excepciones:
22 22 Si un determinado string no puede ser convertido a numero
23 23 Input:
24 24 str, string al cual se le analiza para determinar si convertible a un numero o no
25 25
26 26 Return:
27 27 True : si el string es uno numerico
28 28 False : no es un string numerico
29 29 """
30 30 try:
31 31 float( str )
32 32 return True
33 33 except:
34 34 return False
35 35
36 36 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
37 37 """
38 38 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
39 39
40 40 Inputs:
41 41 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
42 42
43 43 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
44 44 segundos contados desde 01/01/1970.
45 45 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
46 46 segundos contados desde 01/01/1970.
47 47
48 48 Return:
49 49 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
50 50 fecha especificado, de lo contrario retorna False.
51 51
52 52 Excepciones:
53 53 Si el archivo no existe o no puede ser abierto
54 54 Si la cabecera no puede ser leida.
55 55
56 56 """
57 57 basicHeaderObj = BasicHeader()
58 58
59 59 try:
60 60 fp = open(filename,'rb')
61 61 except:
62 62 raise IOError, "The file %s can't be opened" %(filename)
63 63
64 64 sts = basicHeaderObj.read(fp)
65 65 fp.close()
66 66
67 67 if not(sts):
68 68 print "Skipping the file %s because it has not a valid header" %(filename)
69 69 return 0
70 70
71 71 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
72 72 return 0
73 73
74 74 return 1
75 75
76 76 def getlastFileFromPath(path, ext):
77 77 """
78 78 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
79 79 al final de la depuracion devuelve el ultimo file de la lista que quedo.
80 80
81 81 Input:
82 82 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
83 83 ext : extension de los files contenidos en una carpeta
84 84
85 85 Return:
86 86 El ultimo file de una determinada carpeta, no se considera el path.
87 87 """
88 88 validFilelist = []
89 89 fileList = os.listdir(path)
90 90
91 91 # 0 1234 567 89A BCDE
92 92 # H YYYY DDD SSS .ext
93 93
94 94 for file in fileList:
95 95 try:
96 96 year = int(file[1:5])
97 97 doy = int(file[5:8])
98 98
99 99 if (os.path.splitext(file)[-1].upper() != ext.upper()) : continue
100 100 except:
101 101 continue
102 102
103 103 validFilelist.append(file)
104 104
105 105 if validFilelist:
106 106 validFilelist = sorted( validFilelist, key=str.lower )
107 107 return validFilelist[-1]
108 108
109 109 return None
110 110
111 111 def checkForRealPath(path, year, doy, set, ext):
112 112 """
113 113 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
114 114 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
115 115 el path exacto de un determinado file.
116 116
117 117 Example :
118 118 nombre correcto del file es .../.../D2009307/P2009307367.ext
119 119
120 120 Entonces la funcion prueba con las siguientes combinaciones
121 121 .../.../x2009307/y2009307367.ext
122 122 .../.../x2009307/Y2009307367.ext
123 123 .../.../X2009307/y2009307367.ext
124 124 .../.../X2009307/Y2009307367.ext
125 125 siendo para este caso, la ultima combinacion de letras, identica al file buscado
126 126
127 127 Return:
128 128 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
129 129 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
130 130 para el filename
131 131 """
132 132 filepath = None
133 133 find_flag = False
134 134 filename = None
135 135
136 136 if ext.lower() == ".r": #voltage
137 137 header1 = "dD"
138 138 header2 = "dD"
139 139 elif ext.lower() == ".pdata": #spectra
140 140 header1 = "dD"
141 141 header2 = "pP"
142 142 else:
143 143 return None, filename
144 144
145 145 for dir in header1: #barrido por las dos combinaciones posibles de "D"
146 146 for fil in header2: #barrido por las dos combinaciones posibles de "D"
147 147 doypath = "%s%04d%03d" % ( dir, year, doy ) #formo el nombre del directorio xYYYYDDD (x=d o x=D)
148 148 filename = "%s%04d%03d%03d%s" % ( fil, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
149 149 filepath = os.path.join( path, doypath, filename ) #formo el path completo
150 150 if os.path.exists( filepath ): #verifico que exista
151 151 find_flag = True
152 152 break
153 153 if find_flag:
154 154 break
155 155
156 156 if not(find_flag):
157 157 return None, filename
158 158
159 159 return filepath, filename
160 160
161 161 class JRODataIO:
162 162
163 163 c = 3E8
164 164
165 165 isConfig = False
166 166
167 167 basicHeaderObj = BasicHeader()
168 168
169 169 systemHeaderObj = SystemHeader()
170 170
171 171 radarControllerHeaderObj = RadarControllerHeader()
172 172
173 173 processingHeaderObj = ProcessingHeader()
174 174
175 175 online = 0
176 176
177 177 dtype = None
178 178
179 179 pathList = []
180 180
181 181 filenameList = []
182 182
183 183 filename = None
184 184
185 185 ext = None
186 186
187 187 flagNoMoreFiles = 0
188 188
189 189 flagIsNewFile = 1
190 190
191 191 flagTimeBlock = 0
192 192
193 193 flagIsNewBlock = 0
194 194
195 195 fp = None
196 196
197 197 firstHeaderSize = 0
198 198
199 199 basicHeaderSize = 24
200 200
201 201 versionFile = 1103
202 202
203 203 fileSize = None
204 204
205 205 ippSeconds = None
206 206
207 207 fileSizeByHeader = None
208 208
209 209 fileIndex = None
210 210
211 211 profileIndex = None
212 212
213 213 blockIndex = None
214 214
215 215 nTotalBlocks = None
216 216
217 217 maxTimeStep = 30
218 218
219 219 lastUTTime = None
220 220
221 221 datablock = None
222 222
223 223 dataOut = None
224 224
225 225 blocksize = None
226 226
227 227 def __init__(self):
228 228
229 229 raise ValueError, "Not implemented"
230 230
231 231 def run(self):
232 232
233 233 raise ValueError, "Not implemented"
234 234
235 def getOuput(self):
235 236
237 return self.dataOut
236 238
237 239 class JRODataReader(JRODataIO):
238 240
239 241 nReadBlocks = 0
240 242
241 243 delay = 60 #number of seconds waiting a new file
242 244
243 245 nTries = 3 #quantity tries
244 246
245 247 nFiles = 3 #number of files for searching
246 248
247 249
248 250 def __init__(self):
249 251
250 252 """
251 253
252 254 """
253 255
254 256 raise ValueError, "This method has not been implemented"
255 257
256 258
257 259 def createObjByDefault(self):
258 260 """
259 261
260 262 """
261 263 raise ValueError, "This method has not been implemented"
262 264
263 265 def getBlockDimension(self):
264 266
265 267 raise ValueError, "No implemented"
266 268
267 269 def __searchFilesOffLine(self,
268 270 path,
269 271 startDate,
270 272 endDate,
271 273 startTime=datetime.time(0,0,0),
272 274 endTime=datetime.time(23,59,59),
273 275 set=None,
274 276 expLabel="",
275 277 ext=".r"):
276 278 dirList = []
277 279 for thisPath in os.listdir(path):
278 280 if os.path.isdir(os.path.join(path,thisPath)):
279 281 dirList.append(thisPath)
280 282
281 283 if not(dirList):
282 284 return None, None
283 285
284 286 pathList = []
285 287 dateList = []
286 288
287 289 thisDate = startDate
288 290
289 291 while(thisDate <= endDate):
290 292 year = thisDate.timetuple().tm_year
291 293 doy = thisDate.timetuple().tm_yday
292 294
293 295 match = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy))
294 296 if len(match) == 0:
295 297 thisDate += datetime.timedelta(1)
296 298 continue
297 299
298 300 pathList.append(os.path.join(path,match[0],expLabel))
299 301 dateList.append(thisDate)
300 302 thisDate += datetime.timedelta(1)
301 303
302 304 filenameList = []
303 305 for index in range(len(pathList)):
304 306
305 307 thisPath = pathList[index]
306 308 fileList = glob.glob1(thisPath, "*%s" %ext)
307 309 fileList.sort()
308 310
309 311 #Busqueda de datos en el rango de horas indicados
310 312 thisDate = dateList[index]
311 313 startDT = datetime.datetime.combine(thisDate, startTime)
312 314 endDT = datetime.datetime.combine(thisDate, endTime)
313 315
314 316 startUtSeconds = time.mktime(startDT.timetuple())
315 317 endUtSeconds = time.mktime(endDT.timetuple())
316 318
317 319 for file in fileList:
318 320
319 321 filename = os.path.join(thisPath,file)
320 322
321 323 if isThisFileinRange(filename, startUtSeconds, endUtSeconds):
322 324 filenameList.append(filename)
323 325
324 326 if not(filenameList):
325 327 return None, None
326 328
327 329 self.filenameList = filenameList
328 330
329 331 return pathList, filenameList
330 332
331 333 def __searchFilesOnLine(self, path, startDate=None, endDate=None, startTime=None, endTime=None, expLabel = "", ext = None):
332 334
333 335 """
334 336 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
335 337 devuelve el archivo encontrado ademas de otros datos.
336 338
337 339 Input:
338 340 path : carpeta donde estan contenidos los files que contiene data
339 341
340 342 startDate : Fecha inicial. Rechaza todos los directorios donde
341 343 file end time < startDate (obejto datetime.date)
342 344
343 345 endDate : Fecha final. Rechaza todos los directorios donde
344 346 file start time > endDate (obejto datetime.date)
345 347
346 348 startTime : Tiempo inicial. Rechaza todos los archivos donde
347 349 file end time < startTime (obejto datetime.time)
348 350
349 351 endTime : Tiempo final. Rechaza todos los archivos donde
350 352 file start time > endTime (obejto datetime.time)
351 353
352 354 expLabel : Nombre del subexperimento (subfolder)
353 355
354 356 ext : extension de los files
355 357
356 358 Return:
357 359 directory : eL directorio donde esta el file encontrado
358 360 filename : el ultimo file de una determinada carpeta
359 361 year : el anho
360 362 doy : el numero de dia del anho
361 363 set : el set del archivo
362 364
363 365
364 366 """
365 367 dirList = []
366 368 pathList = []
367 369 directory = None
368 370
369 371 #Filtra solo los directorios
370 372 for thisPath in os.listdir(path):
371 373 if os.path.isdir(os.path.join(path, thisPath)):
372 374 dirList.append(thisPath)
373 375
374 376 if not(dirList):
375 377 return None, None, None, None, None
376 378
377 379 dirList = sorted( dirList, key=str.lower )
378 380
379 381 if startDate:
380 382 startDateTime = datetime.datetime.combine(startDate, startTime)
381 383 thisDateTime = startDateTime
382 384 if endDate == None: endDateTime = startDateTime
383 385 else: endDateTime = datetime.datetime.combine(endDate, endTime)
384 386
385 387 while(thisDateTime <= endDateTime):
386 388 year = thisDateTime.timetuple().tm_year
387 389 doy = thisDateTime.timetuple().tm_yday
388 390
389 391 match = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy))
390 392 if len(match) == 0:
391 393 thisDateTime += datetime.timedelta(1)
392 394 continue
393 395
394 396 pathList.append(os.path.join(path,match[0], expLabel))
395 397 thisDateTime += datetime.timedelta(1)
396 398
397 399 if not(pathList):
398 400 print "\tNo files in range: %s - %s" %(startDateTime.ctime(), endDateTime.ctime())
399 401 return None, None, None, None, None
400 402
401 403 directory = pathList[0]
402 404
403 405 else:
404 406 directory = dirList[-1]
405 407 directory = os.path.join(path,directory)
406 408
407 409 filename = getlastFileFromPath(directory, ext)
408 410
409 411 if not(filename):
410 412 return None, None, None, None, None
411 413
412 414 if not(self.__verifyFile(os.path.join(directory, filename))):
413 415 return None, None, None, None, None
414 416
415 417 year = int( filename[1:5] )
416 418 doy = int( filename[5:8] )
417 419 set = int( filename[8:11] )
418 420
419 421 return directory, filename, year, doy, set
420 422
421 423 def setup(self,
422 424 path=None,
423 425 startDate=None,
424 426 endDate=None,
425 427 startTime=datetime.time(0,0,0),
426 428 endTime=datetime.time(23,59,59),
427 429 set=0,
428 430 expLabel = "",
429 431 ext = None,
430 432 online = False,
431 433 delay = 60):
432 434
433 435 if path == None:
434 436 raise ValueError, "The path is not valid"
435 437
436 438 if ext == None:
437 439 ext = self.ext
438 440
439 441 if online:
440 442 print "Searching files in online mode..."
441 443 doypath, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext)
442 444
443 445 if not(doypath):
444 446 for nTries in range( self.nTries ):
445 447 print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
446 448 time.sleep( self.delay )
447 449 doypath, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=exp)
448 450 if doypath:
449 451 break
450 452
451 453 if not(doypath):
452 454 print "There 'isn't valied files in %s" % path
453 455 return None
454 456
455 457 self.year = year
456 458 self.doy = doy
457 459 self.set = set - 1
458 460 self.path = path
459 461
460 462 else:
461 463 print "Searching files in offline mode ..."
462 464 pathList, filenameList = self.__searchFilesOffLine(path, startDate, endDate, startTime, endTime, set, expLabel, ext)
463 465
464 466 if not(pathList):
465 467 print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
466 468 datetime.datetime.combine(startDate,startTime).ctime(),
467 469 datetime.datetime.combine(endDate,endTime).ctime())
468 470
469 471 sys.exit(-1)
470 472
471 473
472 474 self.fileIndex = -1
473 475 self.pathList = pathList
474 476 self.filenameList = filenameList
475 477
476 478 self.online = online
477 479 self.delay = delay
478 480 ext = ext.lower()
479 481 self.ext = ext
480 482
481 483 if not(self.setNextFile()):
482 484 if (startDate!=None) and (endDate!=None):
483 485 print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
484 486 elif startDate != None:
485 487 print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
486 488 else:
487 489 print "No files"
488 490
489 491 sys.exit(-1)
490 492
491 493 # self.updateDataHeader()
492 494
493 495 return self.dataOut
494 496
495 497 def __setNextFileOffline(self):
496 498
497 499 idFile = self.fileIndex
498 500
499 501 while (True):
500 502 idFile += 1
501 503 if not(idFile < len(self.filenameList)):
502 504 self.flagNoMoreFiles = 1
503 505 print "No more Files"
504 506 return 0
505 507
506 508 filename = self.filenameList[idFile]
507 509
508 510 if not(self.__verifyFile(filename)):
509 511 continue
510 512
511 513 fileSize = os.path.getsize(filename)
512 514 fp = open(filename,'rb')
513 515 break
514 516
515 517 self.flagIsNewFile = 1
516 518 self.fileIndex = idFile
517 519 self.filename = filename
518 520 self.fileSize = fileSize
519 521 self.fp = fp
520 522
521 523 print "Setting the file: %s"%self.filename
522 524
523 525 return 1
524 526
525 527 def __setNextFileOnline(self):
526 528 """
527 529 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
528 530 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
529 531 siguientes.
530 532
531 533 Affected:
532 534 self.flagIsNewFile
533 535 self.filename
534 536 self.fileSize
535 537 self.fp
536 538 self.set
537 539 self.flagNoMoreFiles
538 540
539 541 Return:
540 542 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
541 543 1 : si el file fue abierto con exito y esta listo a ser leido
542 544
543 545 Excepciones:
544 546 Si un determinado file no puede ser abierto
545 547 """
546 548 nFiles = 0
547 549 fileOk_flag = False
548 550 firstTime_flag = True
549 551
550 552 self.set += 1
551 553
552 554 #busca el 1er file disponible
553 555 file, filename = checkForRealPath( self.path, self.year, self.doy, self.set, self.ext )
554 556 if file:
555 557 if self.__verifyFile(file, False):
556 558 fileOk_flag = True
557 559
558 560 #si no encuentra un file entonces espera y vuelve a buscar
559 561 if not(fileOk_flag):
560 562 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
561 563
562 564 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
563 565 tries = self.nTries
564 566 else:
565 567 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
566 568
567 569 for nTries in range( tries ):
568 570 if firstTime_flag:
569 571 print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
570 572 time.sleep( self.delay )
571 573 else:
572 574 print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
573 575
574 576 file, filename = checkForRealPath( self.path, self.year, self.doy, self.set, self.ext )
575 577 if file:
576 578 if self.__verifyFile(file):
577 579 fileOk_flag = True
578 580 break
579 581
580 582 if fileOk_flag:
581 583 break
582 584
583 585 firstTime_flag = False
584 586
585 587 print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename
586 588 self.set += 1
587 589
588 590 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
589 591 self.set = 0
590 592 self.doy += 1
591 593
592 594 if fileOk_flag:
593 595 self.fileSize = os.path.getsize( file )
594 596 self.filename = file
595 597 self.flagIsNewFile = 1
596 598 if self.fp != None: self.fp.close()
597 599 self.fp = open(file)
598 600 self.flagNoMoreFiles = 0
599 601 print 'Setting the file: %s' % file
600 602 else:
601 603 self.fileSize = 0
602 604 self.filename = None
603 605 self.flagIsNewFile = 0
604 606 self.fp = None
605 607 self.flagNoMoreFiles = 1
606 608 print 'No more Files'
607 609
608 610 return fileOk_flag
609 611
610 612
611 613 def setNextFile(self):
612 614 if self.fp != None:
613 615 self.fp.close()
614 616
615 617 if self.online:
616 618 newFile = self.__setNextFileOnline()
617 619 else:
618 620 newFile = self.__setNextFileOffline()
619 621
620 622 if not(newFile):
621 623 return 0
622 624
623 625 self.__readFirstHeader()
624 626 self.nReadBlocks = 0
625 627 return 1
626 628
627 629 def __setNewBlock(self):
628 630 if self.fp == None:
629 631 return 0
630 632
631 633 if self.flagIsNewFile:
632 634 return 1
633 635
634 636 self.lastUTTime = self.basicHeaderObj.utc
635 637 currentSize = self.fileSize - self.fp.tell()
636 638 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
637 639
638 640 if (currentSize >= neededSize):
639 641 self.__rdBasicHeader()
640 642 return 1
641 643
642 644 if not(self.setNextFile()):
643 645 return 0
644 646
645 647 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
646 648
647 649 self.flagTimeBlock = 0
648 650
649 651 if deltaTime > self.maxTimeStep:
650 652 self.flagTimeBlock = 1
651 653
652 654 return 1
653 655
654 656
655 657 def readNextBlock(self):
656 658 if not(self.__setNewBlock()):
657 659 return 0
658 660
659 661 if not(self.readBlock()):
660 662 return 0
661 663
662 664 return 1
663 665
664 666 def __rdProcessingHeader(self, fp=None):
665 667 if fp == None:
666 668 fp = self.fp
667 669
668 670 self.processingHeaderObj.read(fp)
669 671
670 672 def __rdRadarControllerHeader(self, fp=None):
671 673 if fp == None:
672 674 fp = self.fp
673 675
674 676 self.radarControllerHeaderObj.read(fp)
675 677
676 678 def __rdSystemHeader(self, fp=None):
677 679 if fp == None:
678 680 fp = self.fp
679 681
680 682 self.systemHeaderObj.read(fp)
681 683
682 684 def __rdBasicHeader(self, fp=None):
683 685 if fp == None:
684 686 fp = self.fp
685 687
686 688 self.basicHeaderObj.read(fp)
687 689
688 690
689 691 def __readFirstHeader(self):
690 692 self.__rdBasicHeader()
691 693 self.__rdSystemHeader()
692 694 self.__rdRadarControllerHeader()
693 695 self.__rdProcessingHeader()
694 696
695 697 self.firstHeaderSize = self.basicHeaderObj.size
696 698
697 699 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
698 700 if datatype == 0:
699 701 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
700 702 elif datatype == 1:
701 703 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
702 704 elif datatype == 2:
703 705 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
704 706 elif datatype == 3:
705 707 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
706 708 elif datatype == 4:
707 709 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
708 710 elif datatype == 5:
709 711 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
710 712 else:
711 713 raise ValueError, 'Data type was not defined'
712 714
713 715 self.dtype = datatype_str
714 716 self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
715 717 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
716 718 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
717 719 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
718 720 self.getBlockDimension()
719 721
720 722
721 723 def __verifyFile(self, filename, msgFlag=True):
722 724 msg = None
723 725 try:
724 726 fp = open(filename, 'rb')
725 727 currentPosition = fp.tell()
726 728 except:
727 729 if msgFlag:
728 730 print "The file %s can't be opened" % (filename)
729 731 return False
730 732
731 733 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
732 734
733 735 if neededSize == 0:
734 736 basicHeaderObj = BasicHeader()
735 737 systemHeaderObj = SystemHeader()
736 738 radarControllerHeaderObj = RadarControllerHeader()
737 739 processingHeaderObj = ProcessingHeader()
738 740
739 741 try:
740 742 if not( basicHeaderObj.read(fp) ): raise ValueError
741 743 if not( systemHeaderObj.read(fp) ): raise ValueError
742 744 if not( radarControllerHeaderObj.read(fp) ): raise ValueError
743 745 if not( processingHeaderObj.read(fp) ): raise ValueError
744 746 data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
745 747
746 748 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
747 749
748 750 except:
749 751 if msgFlag:
750 752 print "\tThe file %s is empty or it hasn't enough data" % filename
751 753
752 754 fp.close()
753 755 return False
754 756 else:
755 757 msg = "\tSkipping the file %s due to it hasn't enough data" %filename
756 758
757 759 fp.close()
758 760 fileSize = os.path.getsize(filename)
759 761 currentSize = fileSize - currentPosition
760 762 if currentSize < neededSize:
761 763 if msgFlag and (msg != None):
762 764 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
763 765 return False
764 766
765 767 return True
766 768
767 769 def getData():
768 770 pass
769 771
770 772 def hasNotDataInBuffer():
771 773 pass
772 774
773 775 def readBlock():
774 776 pass
775 777
776 778 def run(self, **kwargs):
777 779
778 780 if not(self.isConfig):
779 781
780 782 # self.dataOut = dataOut
781 783 self.setup(**kwargs)
782 784 self.isConfig = True
783 785
784 786 self.getData()
785 787
786 788 class JRODataWriter(JRODataIO):
787 789
788 790 """
789 791 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
790 792 de los datos siempre se realiza por bloques.
791 793 """
792 794
793 795 blockIndex = 0
794 796
795 797 path = None
796 798
797 799 setFile = None
798 800
799 801 profilesPerBlock = None
800 802
801 803 blocksPerFile = None
802 804
803 805 nWriteBlocks = 0
804 806
805 807 def __init__(self, dataOut=None):
806 808 raise ValueError, "Not implemented"
807 809
808 810
809 811 def hasAllDataInBuffer(self):
810 812 raise ValueError, "Not implemented"
811 813
812 814
813 815 def setBlockDimension(self):
814 816 raise ValueError, "Not implemented"
815 817
816 818
817 819 def writeBlock(self):
818 820 raise ValueError, "No implemented"
819 821
820 822
821 823 def putData(self):
822 824 raise ValueError, "No implemented"
823 825
824 826 def getDataHeader(self):
825 827 """
826 828 Obtiene una copia del First Header
827 829
828 830 Affected:
829 831
830 832 self.basicHeaderObj
831 833 self.systemHeaderObj
832 834 self.radarControllerHeaderObj
833 835 self.processingHeaderObj self.
834 836
835 837 Return:
836 838 None
837 839 """
838 840
839 841 raise ValueError, "No implemented"
840 842
841 843 def getBasicHeader(self):
842 844
843 845 self.basicHeaderObj.size = self.basicHeaderSize #bytes
844 846 self.basicHeaderObj.version = self.versionFile
845 847 self.basicHeaderObj.dataBlock = self.nTotalBlocks
846 848
847 849 utc = numpy.floor(self.dataOut.utctime)
848 850 milisecond = (self.dataOut.utctime - utc)* 1000.0
849 851
850 852 self.basicHeaderObj.utc = utc
851 853 self.basicHeaderObj.miliSecond = milisecond
852 854 self.basicHeaderObj.timeZone = 0
853 855 self.basicHeaderObj.dstFlag = 0
854 856 self.basicHeaderObj.errorCount = 0
855 857
856 858 def __writeFirstHeader(self):
857 859 """
858 860 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
859 861
860 862 Affected:
861 863 __dataType
862 864
863 865 Return:
864 866 None
865 867 """
866 868
867 869 # CALCULAR PARAMETROS
868 870
869 871 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
870 872 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
871 873
872 874 self.basicHeaderObj.write(self.fp)
873 875 self.systemHeaderObj.write(self.fp)
874 876 self.radarControllerHeaderObj.write(self.fp)
875 877 self.processingHeaderObj.write(self.fp)
876 878
877 879 self.dtype = self.dataOut.dtype
878 880
879 881 def __setNewBlock(self):
880 882 """
881 883 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
882 884
883 885 Return:
884 886 0 : si no pudo escribir nada
885 887 1 : Si escribio el Basic el First Header
886 888 """
887 889 if self.fp == None:
888 890 self.setNextFile()
889 891
890 892 if self.flagIsNewFile:
891 893 return 1
892 894
893 895 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
894 896 self.basicHeaderObj.write(self.fp)
895 897 return 1
896 898
897 899 if not( self.setNextFile() ):
898 900 return 0
899 901
900 902 return 1
901 903
902 904
903 905 def writeNextBlock(self):
904 906 """
905 907 Selecciona el bloque siguiente de datos y los escribe en un file
906 908
907 909 Return:
908 910 0 : Si no hizo pudo escribir el bloque de datos
909 911 1 : Si no pudo escribir el bloque de datos
910 912 """
911 913 if not( self.__setNewBlock() ):
912 914 return 0
913 915
914 916 self.writeBlock()
915 917
916 918 return 1
917 919
918 920 def setNextFile(self):
919 921 """
920 922 Determina el siguiente file que sera escrito
921 923
922 924 Affected:
923 925 self.filename
924 926 self.subfolder
925 927 self.fp
926 928 self.setFile
927 929 self.flagIsNewFile
928 930
929 931 Return:
930 932 0 : Si el archivo no puede ser escrito
931 933 1 : Si el archivo esta listo para ser escrito
932 934 """
933 935 ext = self.ext
934 936 path = self.path
935 937
936 938 if self.fp != None:
937 939 self.fp.close()
938 940
939 941 timeTuple = time.localtime( self.dataOut.dataUtcTime)
940 942 subfolder = 'D%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
941 943
942 944 doypath = os.path.join( path, subfolder )
943 945 if not( os.path.exists(doypath) ):
944 946 os.mkdir(doypath)
945 947 self.setFile = -1 #inicializo mi contador de seteo
946 948 else:
947 949 filesList = os.listdir( doypath )
948 950 if len( filesList ) > 0:
949 951 filesList = sorted( filesList, key=str.lower )
950 952 filen = filesList[-1]
951 953 # el filename debera tener el siguiente formato
952 954 # 0 1234 567 89A BCDE (hex)
953 955 # x YYYY DDD SSS .ext
954 956 if isNumber( filen[8:11] ):
955 957 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
956 958 else:
957 959 self.setFile = -1
958 960 else:
959 961 self.setFile = -1 #inicializo mi contador de seteo
960 962
961 963 setFile = self.setFile
962 964 setFile += 1
963 965
964 966 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
965 967 timeTuple.tm_year,
966 968 timeTuple.tm_yday,
967 969 setFile,
968 970 ext )
969 971
970 972 filename = os.path.join( path, subfolder, file )
971 973
972 974 fp = open( filename,'wb' )
973 975
974 976 self.blockIndex = 0
975 977
976 978 #guardando atributos
977 979 self.filename = filename
978 980 self.subfolder = subfolder
979 981 self.fp = fp
980 982 self.setFile = setFile
981 983 self.flagIsNewFile = 1
982 984
983 985 self.getDataHeader()
984 986
985 987 print 'Writing the file: %s'%self.filename
986 988
987 989 self.__writeFirstHeader()
988 990
989 991 return 1
990 992
991 993 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=None, set=0, ext=None):
992 994 """
993 995 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
994 996
995 997 Inputs:
996 998 path : el path destino en el cual se escribiran los files a crear
997 999 format : formato en el cual sera salvado un file
998 1000 set : el setebo del file
999 1001
1000 1002 Return:
1001 1003 0 : Si no realizo un buen seteo
1002 1004 1 : Si realizo un buen seteo
1003 1005 """
1004 1006
1005 1007 if ext == None:
1006 1008 ext = self.ext
1007 1009
1008 1010 ext = ext.lower()
1009 1011
1010 1012 self.ext = ext
1011 1013
1012 1014 self.path = path
1013 1015
1014 1016 self.setFile = set - 1
1015 1017
1016 1018 self.blocksPerFile = blocksPerFile
1017 1019
1018 1020 self.profilesPerBlock = profilesPerBlock
1019 1021
1020 1022 self.dataOut = dataOut
1021 1023
1022 1024 if not(self.setNextFile()):
1023 1025 print "There isn't a next file"
1024 1026 return 0
1025 1027
1026 1028 self.setBlockDimension()
1027 1029
1028 1030 return 1
1029 1031
1030 1032 def run(self, dataOut, **kwargs):
1031 1033
1032 1034 if not(self.isConfig):
1033 1035
1034 1036 self.setup(dataOut, **kwargs)
1035 1037 self.isConfig = True
1036 1038
1037 1039 self.putData()
1038 1040
1039 1041 class VoltageReader(JRODataReader):
1040 1042 """
1041 1043 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
1042 1044 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
1043 1045 perfiles*alturas*canales) son almacenados en la variable "buffer".
1044 1046
1045 1047 perfiles * alturas * canales
1046 1048
1047 1049 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
1048 1050 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
1049 1051 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
1050 1052 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
1051 1053
1052 1054 Example:
1053 1055
1054 1056 dpath = "/home/myuser/data"
1055 1057
1056 1058 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
1057 1059
1058 1060 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
1059 1061
1060 1062 readerObj = VoltageReader()
1061 1063
1062 1064 readerObj.setup(dpath, startTime, endTime)
1063 1065
1064 1066 while(True):
1065 1067
1066 1068 #to get one profile
1067 1069 profile = readerObj.getData()
1068 1070
1069 1071 #print the profile
1070 1072 print profile
1071 1073
1072 1074 #If you want to see all datablock
1073 1075 print readerObj.datablock
1074 1076
1075 1077 if readerObj.flagNoMoreFiles:
1076 1078 break
1077 1079
1078 1080 """
1079 1081
1080 1082 ext = ".r"
1081 1083
1082 1084 optchar = "D"
1083 1085 dataOut = None
1084 1086
1085 1087
1086 1088 def __init__(self):
1087 1089 """
1088 1090 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
1089 1091
1090 1092 Input:
1091 1093 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
1092 1094 almacenar un perfil de datos cada vez que se haga un requerimiento
1093 1095 (getData). El perfil sera obtenido a partir del buffer de datos,
1094 1096 si el buffer esta vacio se hara un nuevo proceso de lectura de un
1095 1097 bloque de datos.
1096 1098 Si este parametro no es pasado se creara uno internamente.
1097 1099
1098 1100 Variables afectadas:
1099 1101 self.dataOut
1100 1102
1101 1103 Return:
1102 1104 None
1103 1105 """
1104 1106
1105 1107 self.isConfig = False
1106 1108
1107 1109 self.datablock = None
1108 1110
1109 1111 self.utc = 0
1110 1112
1111 1113 self.ext = ".r"
1112 1114
1113 1115 self.optchar = "D"
1114 1116
1115 1117 self.basicHeaderObj = BasicHeader()
1116 1118
1117 1119 self.systemHeaderObj = SystemHeader()
1118 1120
1119 1121 self.radarControllerHeaderObj = RadarControllerHeader()
1120 1122
1121 1123 self.processingHeaderObj = ProcessingHeader()
1122 1124
1123 1125 self.online = 0
1124 1126
1125 1127 self.fp = None
1126 1128
1127 1129 self.idFile = None
1128 1130
1129 1131 self.dtype = None
1130 1132
1131 1133 self.fileSizeByHeader = None
1132 1134
1133 1135 self.filenameList = []
1134 1136
1135 1137 self.filename = None
1136 1138
1137 1139 self.fileSize = None
1138 1140
1139 1141 self.firstHeaderSize = 0
1140 1142
1141 1143 self.basicHeaderSize = 24
1142 1144
1143 1145 self.pathList = []
1144 1146
1145 1147 self.filenameList = []
1146 1148
1147 1149 self.lastUTTime = 0
1148 1150
1149 1151 self.maxTimeStep = 30
1150 1152
1151 1153 self.flagNoMoreFiles = 0
1152 1154
1153 1155 self.set = 0
1154 1156
1155 1157 self.path = None
1156 1158
1157 1159 self.profileIndex = 9999
1158 1160
1159 1161 self.delay = 3 #seconds
1160 1162
1161 1163 self.nTries = 3 #quantity tries
1162 1164
1163 1165 self.nFiles = 3 #number of files for searching
1164 1166
1165 1167 self.nReadBlocks = 0
1166 1168
1167 1169 self.flagIsNewFile = 1
1168 1170
1169 1171 self.ippSeconds = 0
1170 1172
1171 1173 self.flagTimeBlock = 0
1172 1174
1173 1175 self.flagIsNewBlock = 0
1174 1176
1175 1177 self.nTotalBlocks = 0
1176 1178
1177 1179 self.blocksize = 0
1178 1180
1179 1181 self.dataOut = self.createObjByDefault()
1180 1182
1181 1183 def createObjByDefault(self):
1182 1184
1183 1185 dataObj = Voltage()
1184 1186
1185 1187 return dataObj
1186 1188
1187 1189 def __hasNotDataInBuffer(self):
1188 1190 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
1189 1191 return 1
1190 1192 return 0
1191 1193
1192 1194
1193 1195 def getBlockDimension(self):
1194 1196 """
1195 1197 Obtiene la cantidad de puntos a leer por cada bloque de datos
1196 1198
1197 1199 Affected:
1198 1200 self.blocksize
1199 1201
1200 1202 Return:
1201 1203 None
1202 1204 """
1203 1205 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
1204 1206 self.blocksize = pts2read
1205 1207
1206 1208
1207 1209 def readBlock(self):
1208 1210 """
1209 1211 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
1210 1212 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
1211 1213 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
1212 1214 es seteado a 0
1213 1215
1214 1216 Inputs:
1215 1217 None
1216 1218
1217 1219 Return:
1218 1220 None
1219 1221
1220 1222 Affected:
1221 1223 self.profileIndex
1222 1224 self.datablock
1223 1225 self.flagIsNewFile
1224 1226 self.flagIsNewBlock
1225 1227 self.nTotalBlocks
1226 1228
1227 1229 Exceptions:
1228 1230 Si un bloque leido no es un bloque valido
1229 1231 """
1230 1232
1231 1233 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
1232 1234
1233 1235 try:
1234 1236 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
1235 1237 except:
1236 1238 print "The read block (%3d) has not enough data" %self.nReadBlocks
1237 1239 return 0
1238 1240
1239 1241 junk = numpy.transpose(junk, (2,0,1))
1240 1242 self.datablock = junk['real'] + junk['imag']*1j
1241 1243
1242 1244 self.profileIndex = 0
1243 1245
1244 1246 self.flagIsNewFile = 0
1245 1247 self.flagIsNewBlock = 1
1246 1248
1247 1249 self.nTotalBlocks += 1
1248 1250 self.nReadBlocks += 1
1249 1251
1250 1252 return 1
1251 1253
1252 1254
1253 1255 def getData(self):
1254 1256 """
1255 1257 getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"
1256 1258 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
1257 1259 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
1258 1260
1259 1261 Ademas incrementa el contador del buffer en 1.
1260 1262
1261 1263 Return:
1262 1264 data : retorna un perfil de voltages (alturas * canales) copiados desde el
1263 1265 buffer. Si no hay mas archivos a leer retorna None.
1264 1266
1265 1267 Variables afectadas:
1266 1268 self.dataOut
1267 1269 self.profileIndex
1268 1270
1269 1271 Affected:
1270 1272 self.dataOut
1271 1273 self.profileIndex
1272 1274 self.flagTimeBlock
1273 1275 self.flagIsNewBlock
1274 1276 """
1275 1277 if self.flagNoMoreFiles: return 0
1276 1278
1277 1279 self.flagTimeBlock = 0
1278 1280 self.flagIsNewBlock = 0
1279 1281
1280 1282 if self.__hasNotDataInBuffer():
1281 1283
1282 1284 if not( self.readNextBlock() ):
1283 1285 return 0
1284 1286
1285 1287 # self.updateDataHeader()
1286 1288
1287 1289 if self.flagNoMoreFiles == 1:
1288 1290 print 'Process finished'
1289 1291 return 0
1290 1292
1291 1293 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
1292 1294
1293 1295 if self.datablock == None:
1294 1296 self.dataOut.flagNoData = True
1295 1297 return 0
1296 1298
1297 1299 self.dataOut.data = self.datablock[:,self.profileIndex,:]
1298 1300
1299 1301 self.dataOut.dtype = self.dtype
1300 1302
1301 1303 self.dataOut.nChannels = self.systemHeaderObj.nChannels
1302 1304
1303 1305 self.dataOut.nHeights = self.processingHeaderObj.nHeights
1304 1306
1305 1307 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
1306 1308
1307 1309 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
1308 1310
1309 1311 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
1310 1312
1311 1313 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
1312 1314
1313 1315 self.dataOut.channelIndexList = range(self.systemHeaderObj.nChannels)
1314 1316
1315 1317 self.dataOut.flagTimeBlock = self.flagTimeBlock
1316 1318
1317 1319 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds
1318 1320
1319 1321 self.dataOut.ippSeconds = self.ippSeconds
1320 1322
1321 1323 self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt
1322 1324
1323 1325 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
1324 1326
1325 1327 self.dataOut.flagShiftFFT = False
1326 1328
1327 1329 if self.processingHeaderObj.code != None:
1328 1330 self.dataOut.nCode = self.processingHeaderObj.nCode
1329 1331
1330 1332 self.dataOut.nBaud = self.processingHeaderObj.nBaud
1331 1333
1332 1334 self.dataOut.code = self.processingHeaderObj.code
1333 1335
1334 1336 self.profileIndex += 1
1335 1337
1336 1338 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
1337 1339
1338 1340 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
1339 1341
1340 1342 self.dataOut.flagNoData = False
1341 1343
1342 1344 # print self.profileIndex, self.dataOut.utctime
1343 1345 # if self.profileIndex == 800:
1344 1346 # a=1
1345 1347
1346 1348 return self.dataOut.data
1347 1349
1348 1350
1349 1351 class VoltageWriter(JRODataWriter):
1350 1352 """
1351 1353 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
1352 1354 de los datos siempre se realiza por bloques.
1353 1355 """
1354 1356
1355 1357 ext = ".r"
1356 1358
1357 1359 optchar = "D"
1358 1360
1359 1361 shapeBuffer = None
1360 1362
1361 1363
1362 1364 def __init__(self):
1363 1365 """
1364 1366 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
1365 1367
1366 1368 Affected:
1367 1369 self.dataOut
1368 1370
1369 1371 Return: None
1370 1372 """
1371 1373
1372 1374 self.nTotalBlocks = 0
1373 1375
1374 1376 self.profileIndex = 0
1375 1377
1376 1378 self.isConfig = False
1377 1379
1378 1380 self.fp = None
1379 1381
1380 1382 self.flagIsNewFile = 1
1381 1383
1382 1384 self.nTotalBlocks = 0
1383 1385
1384 1386 self.flagIsNewBlock = 0
1385 1387
1386 1388 self.flagNoMoreFiles = 0
1387 1389
1388 1390 self.setFile = None
1389 1391
1390 1392 self.dtype = None
1391 1393
1392 1394 self.path = None
1393 1395
1394 1396 self.noMoreFiles = 0
1395 1397
1396 1398 self.filename = None
1397 1399
1398 1400 self.basicHeaderObj = BasicHeader()
1399 1401
1400 1402 self.systemHeaderObj = SystemHeader()
1401 1403
1402 1404 self.radarControllerHeaderObj = RadarControllerHeader()
1403 1405
1404 1406 self.processingHeaderObj = ProcessingHeader()
1405 1407
1406 1408 def hasAllDataInBuffer(self):
1407 1409 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
1408 1410 return 1
1409 1411 return 0
1410 1412
1411 1413
1412 1414 def setBlockDimension(self):
1413 1415 """
1414 1416 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
1415 1417
1416 1418 Affected:
1417 1419 self.shape_spc_Buffer
1418 1420 self.shape_cspc_Buffer
1419 1421 self.shape_dc_Buffer
1420 1422
1421 1423 Return: None
1422 1424 """
1423 1425 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
1424 1426 self.processingHeaderObj.nHeights,
1425 1427 self.systemHeaderObj.nChannels)
1426 1428
1427 1429 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
1428 1430 self.processingHeaderObj.profilesPerBlock,
1429 1431 self.processingHeaderObj.nHeights),
1430 1432 dtype=numpy.dtype('complex'))
1431 1433
1432 1434
1433 1435 def writeBlock(self):
1434 1436 """
1435 1437 Escribe el buffer en el file designado
1436 1438
1437 1439 Affected:
1438 1440 self.profileIndex
1439 1441 self.flagIsNewFile
1440 1442 self.flagIsNewBlock
1441 1443 self.nTotalBlocks
1442 1444 self.blockIndex
1443 1445
1444 1446 Return: None
1445 1447 """
1446 1448 data = numpy.zeros( self.shapeBuffer, self.dtype )
1447 1449
1448 1450 junk = numpy.transpose(self.datablock, (1,2,0))
1449 1451
1450 1452 data['real'] = junk.real
1451 1453 data['imag'] = junk.imag
1452 1454
1453 1455 data = data.reshape( (-1) )
1454 1456
1455 1457 data.tofile( self.fp )
1456 1458
1457 1459 self.datablock.fill(0)
1458 1460
1459 1461 self.profileIndex = 0
1460 1462 self.flagIsNewFile = 0
1461 1463 self.flagIsNewBlock = 1
1462 1464
1463 1465 self.blockIndex += 1
1464 1466 self.nTotalBlocks += 1
1465 1467
1466 1468 def putData(self):
1467 1469 """
1468 1470 Setea un bloque de datos y luego los escribe en un file
1469 1471
1470 1472 Affected:
1471 1473 self.flagIsNewBlock
1472 1474 self.profileIndex
1473 1475
1474 1476 Return:
1475 1477 0 : Si no hay data o no hay mas files que puedan escribirse
1476 1478 1 : Si se escribio la data de un bloque en un file
1477 1479 """
1478 1480 if self.dataOut.flagNoData:
1479 1481 return 0
1480 1482
1481 1483 self.flagIsNewBlock = 0
1482 1484
1483 1485 if self.dataOut.flagTimeBlock:
1484 1486
1485 1487 self.datablock.fill(0)
1486 1488 self.profileIndex = 0
1487 1489 self.setNextFile()
1488 1490
1489 1491 if self.profileIndex == 0:
1490 1492 self.getBasicHeader()
1491 1493
1492 1494 self.datablock[:,self.profileIndex,:] = self.dataOut.data
1493 1495
1494 1496 self.profileIndex += 1
1495 1497
1496 1498 if self.hasAllDataInBuffer():
1497 1499 #if self.flagIsNewFile:
1498 1500 self.writeNextBlock()
1499 1501 # self.getDataHeader()
1500 1502
1501 1503 if self.flagNoMoreFiles:
1502 1504 #print 'Process finished'
1503 1505 return 0
1504 1506
1505 1507 return 1
1506 1508
1507 1509 def __getProcessFlags(self):
1508 1510
1509 1511 processFlags = 0
1510 1512
1511 1513 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
1512 1514 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
1513 1515 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
1514 1516 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
1515 1517 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
1516 1518 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
1517 1519
1518 1520 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
1519 1521
1520 1522
1521 1523
1522 1524 datatypeValueList = [PROCFLAG.DATATYPE_CHAR,
1523 1525 PROCFLAG.DATATYPE_SHORT,
1524 1526 PROCFLAG.DATATYPE_LONG,
1525 1527 PROCFLAG.DATATYPE_INT64,
1526 1528 PROCFLAG.DATATYPE_FLOAT,
1527 1529 PROCFLAG.DATATYPE_DOUBLE]
1528 1530
1529 1531
1530 1532 for index in range(len(dtypeList)):
1531 1533 if self.dataOut.dtype == dtypeList[index]:
1532 1534 dtypeValue = datatypeValueList[index]
1533 1535 break
1534 1536
1535 1537 processFlags += dtypeValue
1536 1538
1537 1539 if self.dataOut.flagDecodeData:
1538 1540 processFlags += PROCFLAG.DECODE_DATA
1539 1541
1540 1542 if self.dataOut.flagDeflipData:
1541 1543 processFlags += PROCFLAG.DEFLIP_DATA
1542 1544
1543 1545 if self.dataOut.code != None:
1544 1546 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1545 1547
1546 1548 if self.dataOut.nCohInt > 1:
1547 1549 processFlags += PROCFLAG.COHERENT_INTEGRATION
1548 1550
1549 1551 return processFlags
1550 1552
1551 1553
1552 1554 def __getBlockSize(self):
1553 1555 '''
1554 1556 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
1555 1557 '''
1556 1558
1557 1559 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
1558 1560 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
1559 1561 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
1560 1562 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
1561 1563 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
1562 1564 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
1563 1565
1564 1566 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
1565 1567 datatypeValueList = [1,2,4,8,4,8]
1566 1568 for index in range(len(dtypeList)):
1567 1569 if self.dataOut.dtype == dtypeList[index]:
1568 1570 datatypeValue = datatypeValueList[index]
1569 1571 break
1570 1572
1571 1573 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.dataOut.nProfiles * datatypeValue * 2)
1572 1574
1573 1575 return blocksize
1574 1576
1575 1577 def getDataHeader(self):
1576 1578
1577 1579 """
1578 1580 Obtiene una copia del First Header
1579 1581
1580 1582 Affected:
1581 1583 self.systemHeaderObj
1582 1584 self.radarControllerHeaderObj
1583 1585 self.dtype
1584 1586
1585 1587 Return:
1586 1588 None
1587 1589 """
1588 1590
1589 1591 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
1590 1592 self.systemHeaderObj.nChannels = self.dataOut.nChannels
1591 1593 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
1592 1594
1593 1595 self.getBasicHeader()
1594 1596
1595 1597 processingHeaderSize = 40 # bytes
1596 1598 self.processingHeaderObj.dtype = 0 # Voltage
1597 1599 self.processingHeaderObj.blockSize = self.__getBlockSize()
1598 1600 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
1599 1601 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
1600 1602 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
1601 1603 self.processingHeaderObj.processFlags = self.__getProcessFlags()
1602 1604 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
1603 1605 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
1604 1606 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
1605 1607
1606 1608 if self.dataOut.code != None:
1607 1609 self.processingHeaderObj.code = self.dataOut.code
1608 1610 self.processingHeaderObj.nCode = self.dataOut.nCode
1609 1611 self.processingHeaderObj.nBaud = self.dataOut.nBaud
1610 1612 codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)
1611 1613 processingHeaderSize += codesize
1612 1614
1613 1615 if self.processingHeaderObj.nWindows != 0:
1614 1616 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
1615 1617 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
1616 1618 self.processingHeaderObj.nHeights = self.dataOut.nHeights
1617 1619 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
1618 1620 processingHeaderSize += 12
1619 1621
1620 1622 self.processingHeaderObj.size = processingHeaderSize
1621 1623
1622 1624 class SpectraReader(JRODataReader):
1623 1625 """
1624 1626 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
1625 1627 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
1626 1628 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
1627 1629
1628 1630 paresCanalesIguales * alturas * perfiles (Self Spectra)
1629 1631 paresCanalesDiferentes * alturas * perfiles (Cross Spectra)
1630 1632 canales * alturas (DC Channels)
1631 1633
1632 1634 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
1633 1635 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
1634 1636 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
1635 1637 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
1636 1638
1637 1639 Example:
1638 1640 dpath = "/home/myuser/data"
1639 1641
1640 1642 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
1641 1643
1642 1644 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
1643 1645
1644 1646 readerObj = SpectraReader()
1645 1647
1646 1648 readerObj.setup(dpath, startTime, endTime)
1647 1649
1648 1650 while(True):
1649 1651
1650 1652 readerObj.getData()
1651 1653
1652 1654 print readerObj.data_spc
1653 1655
1654 1656 print readerObj.data_cspc
1655 1657
1656 1658 print readerObj.data_dc
1657 1659
1658 1660 if readerObj.flagNoMoreFiles:
1659 1661 break
1660 1662
1661 1663 """
1662 1664
1663 1665 pts2read_SelfSpectra = 0
1664 1666
1665 1667 pts2read_CrossSpectra = 0
1666 1668
1667 1669 pts2read_DCchannels = 0
1668 1670
1669 1671 ext = ".pdata"
1670 1672
1671 1673 optchar = "P"
1672 1674
1673 1675 dataOut = None
1674 1676
1675 1677 nRdChannels = None
1676 1678
1677 1679 nRdPairs = None
1678 1680
1679 1681 rdPairList = []
1680 1682
1681 1683
1682 1684 def __init__(self):
1683 1685 """
1684 1686 Inicializador de la clase SpectraReader para la lectura de datos de espectros.
1685 1687
1686 1688 Inputs:
1687 1689 dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para
1688 1690 almacenar un perfil de datos cada vez que se haga un requerimiento
1689 1691 (getData). El perfil sera obtenido a partir del buffer de datos,
1690 1692 si el buffer esta vacio se hara un nuevo proceso de lectura de un
1691 1693 bloque de datos.
1692 1694 Si este parametro no es pasado se creara uno internamente.
1693 1695
1694 1696 Affected:
1695 1697 self.dataOut
1696 1698
1697 1699 Return : None
1698 1700 """
1699 1701
1700 1702 self.isConfig = False
1701 1703
1702 1704 self.pts2read_SelfSpectra = 0
1703 1705
1704 1706 self.pts2read_CrossSpectra = 0
1705 1707
1706 1708 self.pts2read_DCchannels = 0
1707 1709
1708 1710 self.datablock = None
1709 1711
1710 1712 self.utc = None
1711 1713
1712 1714 self.ext = ".pdata"
1713 1715
1714 1716 self.optchar = "P"
1715 1717
1716 1718 self.basicHeaderObj = BasicHeader()
1717 1719
1718 1720 self.systemHeaderObj = SystemHeader()
1719 1721
1720 1722 self.radarControllerHeaderObj = RadarControllerHeader()
1721 1723
1722 1724 self.processingHeaderObj = ProcessingHeader()
1723 1725
1724 1726 self.online = 0
1725 1727
1726 1728 self.fp = None
1727 1729
1728 1730 self.idFile = None
1729 1731
1730 1732 self.dtype = None
1731 1733
1732 1734 self.fileSizeByHeader = None
1733 1735
1734 1736 self.filenameList = []
1735 1737
1736 1738 self.filename = None
1737 1739
1738 1740 self.fileSize = None
1739 1741
1740 1742 self.firstHeaderSize = 0
1741 1743
1742 1744 self.basicHeaderSize = 24
1743 1745
1744 1746 self.pathList = []
1745 1747
1746 1748 self.lastUTTime = 0
1747 1749
1748 1750 self.maxTimeStep = 30
1749 1751
1750 1752 self.flagNoMoreFiles = 0
1751 1753
1752 1754 self.set = 0
1753 1755
1754 1756 self.path = None
1755 1757
1756 1758 self.delay = 3 #seconds
1757 1759
1758 1760 self.nTries = 3 #quantity tries
1759 1761
1760 1762 self.nFiles = 3 #number of files for searching
1761 1763
1762 1764 self.nReadBlocks = 0
1763 1765
1764 1766 self.flagIsNewFile = 1
1765 1767
1766 1768 self.ippSeconds = 0
1767 1769
1768 1770 self.flagTimeBlock = 0
1769 1771
1770 1772 self.flagIsNewBlock = 0
1771 1773
1772 1774 self.nTotalBlocks = 0
1773 1775
1774 1776 self.blocksize = 0
1775 1777
1776 1778 self.dataOut = self.createObjByDefault()
1777 1779
1778 1780
1779 1781 def createObjByDefault(self):
1780 1782
1781 1783 dataObj = Spectra()
1782 1784
1783 1785 return dataObj
1784 1786
1785 1787 def __hasNotDataInBuffer(self):
1786 1788 return 1
1787 1789
1788 1790
1789 1791 def getBlockDimension(self):
1790 1792 """
1791 1793 Obtiene la cantidad de puntos a leer por cada bloque de datos
1792 1794
1793 1795 Affected:
1794 1796 self.nRdChannels
1795 1797 self.nRdPairs
1796 1798 self.pts2read_SelfSpectra
1797 1799 self.pts2read_CrossSpectra
1798 1800 self.pts2read_DCchannels
1799 1801 self.blocksize
1800 1802 self.dataOut.nChannels
1801 1803 self.dataOut.nPairs
1802 1804
1803 1805 Return:
1804 1806 None
1805 1807 """
1806 1808 self.nRdChannels = 0
1807 1809 self.nRdPairs = 0
1808 1810 self.rdPairList = []
1809 1811
1810 1812 for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
1811 1813 if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
1812 1814 self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
1813 1815 else:
1814 1816 self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
1815 1817 self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
1816 1818
1817 1819 pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
1818 1820
1819 1821 self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
1820 1822 self.blocksize = self.pts2read_SelfSpectra
1821 1823
1822 1824 if self.processingHeaderObj.flag_cspc:
1823 1825 self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
1824 1826 self.blocksize += self.pts2read_CrossSpectra
1825 1827
1826 1828 if self.processingHeaderObj.flag_dc:
1827 1829 self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
1828 1830 self.blocksize += self.pts2read_DCchannels
1829 1831
1830 1832 # self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
1831 1833
1832 1834
1833 1835 def readBlock(self):
1834 1836 """
1835 1837 Lee el bloque de datos desde la posicion actual del puntero del archivo
1836 1838 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
1837 1839 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
1838 1840 es seteado a 0
1839 1841
1840 1842 Return: None
1841 1843
1842 1844 Variables afectadas:
1843 1845
1844 1846 self.flagIsNewFile
1845 1847 self.flagIsNewBlock
1846 1848 self.nTotalBlocks
1847 1849 self.data_spc
1848 1850 self.data_cspc
1849 1851 self.data_dc
1850 1852
1851 1853 Exceptions:
1852 1854 Si un bloque leido no es un bloque valido
1853 1855 """
1854 1856 blockOk_flag = False
1855 1857 fpointer = self.fp.tell()
1856 1858
1857 1859 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
1858 1860 spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
1859 1861
1860 1862 if self.processingHeaderObj.flag_cspc:
1861 1863 cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
1862 1864 cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
1863 1865
1864 1866 if self.processingHeaderObj.flag_dc:
1865 1867 dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
1866 1868 dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
1867 1869
1868 1870
1869 1871 if not(self.processingHeaderObj.shif_fft):
1870 1872 spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
1871 1873
1872 1874 if self.processingHeaderObj.flag_cspc:
1873 1875 cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
1874 1876
1875 1877
1876 1878 spc = numpy.transpose( spc, (0,2,1) )
1877 1879 self.data_spc = spc
1878 1880
1879 1881 if self.processingHeaderObj.flag_cspc:
1880 1882 cspc = numpy.transpose( cspc, (0,2,1) )
1881 1883 self.data_cspc = cspc['real'] + cspc['imag']*1j
1882 1884 else:
1883 1885 self.data_cspc = None
1884 1886
1885 1887 if self.processingHeaderObj.flag_dc:
1886 1888 self.data_dc = dc['real'] + dc['imag']*1j
1887 1889 else:
1888 1890 self.data_dc = None
1889 1891
1890 1892 self.flagIsNewFile = 0
1891 1893 self.flagIsNewBlock = 1
1892 1894
1893 1895 self.nTotalBlocks += 1
1894 1896 self.nReadBlocks += 1
1895 1897
1896 1898 return 1
1897 1899
1898 1900
1899 1901 def getData(self):
1900 1902 """
1901 1903 Copia el buffer de lectura a la clase "Spectra",
1902 1904 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
1903 1905 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
1904 1906
1905 1907 Return:
1906 1908 0 : Si no hay mas archivos disponibles
1907 1909 1 : Si hizo una buena copia del buffer
1908 1910
1909 1911 Affected:
1910 1912 self.dataOut
1911 1913
1912 1914 self.flagTimeBlock
1913 1915 self.flagIsNewBlock
1914 1916 """
1915 1917
1916 1918 if self.flagNoMoreFiles: return 0
1917 1919
1918 1920 self.flagTimeBlock = 0
1919 1921 self.flagIsNewBlock = 0
1920 1922
1921 1923 if self.__hasNotDataInBuffer():
1922 1924
1923 1925 if not( self.readNextBlock() ):
1924 1926 return 0
1925 1927
1926 1928 # self.updateDataHeader()
1927 1929
1928 1930 if self.flagNoMoreFiles == 1:
1929 1931 print 'Process finished'
1930 1932 return 0
1931 1933
1932 1934 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
1933 1935
1934 1936 if self.data_dc == None:
1935 1937 self.dataOut.flagNoData = True
1936 1938 return 0
1937 1939
1938 1940
1939 1941 self.dataOut.data_spc = self.data_spc
1940 1942
1941 1943 self.dataOut.data_cspc = self.data_cspc
1942 1944
1943 1945 self.dataOut.data_dc = self.data_dc
1944 1946
1945 1947 self.dataOut.flagTimeBlock = self.flagTimeBlock
1946 1948
1947 1949 self.dataOut.flagNoData = False
1948 1950
1949 1951 self.dataOut.dtype = self.dtype
1950 1952
1951 1953 self.dataOut.nChannels = self.nRdChannels
1952 1954
1953 1955 self.dataOut.nPairs = self.nRdPairs
1954 1956
1955 1957 self.dataOut.pairsList = self.rdPairList
1956 1958
1957 1959 self.dataOut.nHeights = self.processingHeaderObj.nHeights
1958 1960
1959 1961 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
1960 1962
1961 1963 self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
1962 1964
1963 1965 self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
1964 1966
1965 1967
1966 1968 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
1967 1969
1968 1970 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
1969 1971
1970 1972 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
1971 1973
1972 1974 self.dataOut.channelIndexList = range(self.systemHeaderObj.nChannels)
1973 1975
1974 1976 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000.#+ self.profileIndex * self.ippSeconds
1975 1977
1976 1978 self.dataOut.ippSeconds = self.ippSeconds
1977 1979
1978 1980 self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints
1979 1981
1980 1982 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
1981 1983
1982 1984 # self.profileIndex += 1
1983 1985
1984 1986 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
1985 1987
1986 1988 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
1987 1989
1988 1990 return self.dataOut.data_spc
1989 1991
1990 1992
1991 1993 class SpectraWriter(JRODataWriter):
1992 1994
1993 1995 """
1994 1996 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
1995 1997 de los datos siempre se realiza por bloques.
1996 1998 """
1997 1999
1998 2000 ext = ".pdata"
1999 2001
2000 2002 optchar = "P"
2001 2003
2002 2004 shape_spc_Buffer = None
2003 2005
2004 2006 shape_cspc_Buffer = None
2005 2007
2006 2008 shape_dc_Buffer = None
2007 2009
2008 2010 data_spc = None
2009 2011
2010 2012 data_cspc = None
2011 2013
2012 2014 data_dc = None
2013 2015
2014 2016 # dataOut = None
2015 2017
2016 2018 def __init__(self):
2017 2019 """
2018 2020 Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
2019 2021
2020 2022 Affected:
2021 2023 self.dataOut
2022 2024 self.basicHeaderObj
2023 2025 self.systemHeaderObj
2024 2026 self.radarControllerHeaderObj
2025 2027 self.processingHeaderObj
2026 2028
2027 2029 Return: None
2028 2030 """
2029 2031
2030 2032 self.isConfig = False
2031 2033
2032 2034 self.nTotalBlocks = 0
2033 2035
2034 2036 self.data_spc = None
2035 2037
2036 2038 self.data_cspc = None
2037 2039
2038 2040 self.data_dc = None
2039 2041
2040 2042 self.fp = None
2041 2043
2042 2044 self.flagIsNewFile = 1
2043 2045
2044 2046 self.nTotalBlocks = 0
2045 2047
2046 2048 self.flagIsNewBlock = 0
2047 2049
2048 2050 self.flagNoMoreFiles = 0
2049 2051
2050 2052 self.setFile = None
2051 2053
2052 2054 self.dtype = None
2053 2055
2054 2056 self.path = None
2055 2057
2056 2058 self.noMoreFiles = 0
2057 2059
2058 2060 self.filename = None
2059 2061
2060 2062 self.basicHeaderObj = BasicHeader()
2061 2063
2062 2064 self.systemHeaderObj = SystemHeader()
2063 2065
2064 2066 self.radarControllerHeaderObj = RadarControllerHeader()
2065 2067
2066 2068 self.processingHeaderObj = ProcessingHeader()
2067 2069
2068 2070
2069 2071 def hasAllDataInBuffer(self):
2070 2072 return 1
2071 2073
2072 2074
2073 2075 def setBlockDimension(self):
2074 2076 """
2075 2077 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
2076 2078
2077 2079 Affected:
2078 2080 self.shape_spc_Buffer
2079 2081 self.shape_cspc_Buffer
2080 2082 self.shape_dc_Buffer
2081 2083
2082 2084 Return: None
2083 2085 """
2084 2086 self.shape_spc_Buffer = (self.dataOut.nChannels,
2085 2087 self.processingHeaderObj.nHeights,
2086 2088 self.processingHeaderObj.profilesPerBlock)
2087 2089
2088 2090 self.shape_cspc_Buffer = (self.dataOut.nPairs,
2089 2091 self.processingHeaderObj.nHeights,
2090 2092 self.processingHeaderObj.profilesPerBlock)
2091 2093
2092 2094 self.shape_dc_Buffer = (self.dataOut.nChannels,
2093 2095 self.processingHeaderObj.nHeights)
2094 2096
2095 2097
2096 2098 def writeBlock(self):
2097 2099 """
2098 2100 Escribe el buffer en el file designado
2099 2101
2100 2102 Affected:
2101 2103 self.data_spc
2102 2104 self.data_cspc
2103 2105 self.data_dc
2104 2106 self.flagIsNewFile
2105 2107 self.flagIsNewBlock
2106 2108 self.nTotalBlocks
2107 2109 self.nWriteBlocks
2108 2110
2109 2111 Return: None
2110 2112 """
2111 2113
2112 2114 spc = numpy.transpose( self.data_spc, (0,2,1) )
2113 2115 if not( self.processingHeaderObj.shif_fft ):
2114 2116 spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
2115 2117 data = spc.reshape((-1))
2116 2118 data.tofile(self.fp)
2117 2119
2118 2120 if self.data_cspc != None:
2119 2121 data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )
2120 2122 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
2121 2123 if not( self.processingHeaderObj.shif_fft ):
2122 2124 cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
2123 2125 data['real'] = cspc.real
2124 2126 data['imag'] = cspc.imag
2125 2127 data = data.reshape((-1))
2126 2128 data.tofile(self.fp)
2127 2129
2128 2130 if self.data_dc != None:
2129 2131 data = numpy.zeros( self.shape_dc_Buffer, self.dtype )
2130 2132 dc = self.data_dc
2131 2133 data['real'] = dc.real
2132 2134 data['imag'] = dc.imag
2133 2135 data = data.reshape((-1))
2134 2136 data.tofile(self.fp)
2135 2137
2136 2138 self.data_spc.fill(0)
2137 2139 self.data_dc.fill(0)
2138 2140 if self.data_cspc != None:
2139 2141 self.data_cspc.fill(0)
2140 2142
2141 2143 self.flagIsNewFile = 0
2142 2144 self.flagIsNewBlock = 1
2143 2145 self.nTotalBlocks += 1
2144 2146 self.nWriteBlocks += 1
2145 2147 self.blockIndex += 1
2146 2148
2147 2149
2148 2150 def putData(self):
2149 2151 """
2150 2152 Setea un bloque de datos y luego los escribe en un file
2151 2153
2152 2154 Affected:
2153 2155 self.data_spc
2154 2156 self.data_cspc
2155 2157 self.data_dc
2156 2158
2157 2159 Return:
2158 2160 0 : Si no hay data o no hay mas files que puedan escribirse
2159 2161 1 : Si se escribio la data de un bloque en un file
2160 2162 """
2161 2163
2162 2164 if self.dataOut.flagNoData:
2163 2165 return 0
2164 2166
2165 2167 self.flagIsNewBlock = 0
2166 2168
2167 2169 if self.dataOut.flagTimeBlock:
2168 2170 self.data_spc.fill(0)
2169 2171 self.data_cspc.fill(0)
2170 2172 self.data_dc.fill(0)
2171 2173 self.setNextFile()
2172 2174
2173 2175 if self.flagIsNewFile == 0:
2174 2176 self.getBasicHeader()
2175 2177
2176 2178 self.data_spc = self.dataOut.data_spc
2177 2179 self.data_cspc = self.dataOut.data_cspc
2178 2180 self.data_dc = self.dataOut.data_dc
2179 2181
2180 2182 # #self.processingHeaderObj.dataBlocksPerFile)
2181 2183 if self.hasAllDataInBuffer():
2182 2184 # self.getDataHeader()
2183 2185 self.writeNextBlock()
2184 2186
2185 2187 if self.flagNoMoreFiles:
2186 2188 #print 'Process finished'
2187 2189 return 0
2188 2190
2189 2191 return 1
2190 2192
2191 2193
2192 2194 def __getProcessFlags(self):
2193 2195
2194 2196 processFlags = 0
2195 2197
2196 2198 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
2197 2199 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
2198 2200 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
2199 2201 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
2200 2202 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
2201 2203 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
2202 2204
2203 2205 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
2204 2206
2205 2207
2206 2208
2207 2209 datatypeValueList = [PROCFLAG.DATATYPE_CHAR,
2208 2210 PROCFLAG.DATATYPE_SHORT,
2209 2211 PROCFLAG.DATATYPE_LONG,
2210 2212 PROCFLAG.DATATYPE_INT64,
2211 2213 PROCFLAG.DATATYPE_FLOAT,
2212 2214 PROCFLAG.DATATYPE_DOUBLE]
2213 2215
2214 2216
2215 2217 for index in range(len(dtypeList)):
2216 2218 if self.dataOut.dtype == dtypeList[index]:
2217 2219 dtypeValue = datatypeValueList[index]
2218 2220 break
2219 2221
2220 2222 processFlags += dtypeValue
2221 2223
2222 2224 if self.dataOut.flagDecodeData:
2223 2225 processFlags += PROCFLAG.DECODE_DATA
2224 2226
2225 2227 if self.dataOut.flagDeflipData:
2226 2228 processFlags += PROCFLAG.DEFLIP_DATA
2227 2229
2228 2230 if self.dataOut.code != None:
2229 2231 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
2230 2232
2231 2233 if self.dataOut.nIncohInt > 1:
2232 2234 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
2233 2235
2234 2236 if self.dataOut.data_dc != None:
2235 2237 processFlags += PROCFLAG.SAVE_CHANNELS_DC
2236 2238
2237 2239 return processFlags
2238 2240
2239 2241
2240 2242 def __getBlockSize(self):
2241 2243 '''
2242 2244 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
2243 2245 '''
2244 2246
2245 2247 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
2246 2248 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
2247 2249 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
2248 2250 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
2249 2251 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
2250 2252 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
2251 2253
2252 2254 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
2253 2255 datatypeValueList = [1,2,4,8,4,8]
2254 2256 for index in range(len(dtypeList)):
2255 2257 if self.dataOut.dtype == dtypeList[index]:
2256 2258 datatypeValue = datatypeValueList[index]
2257 2259 break
2258 2260
2259 2261
2260 2262 pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
2261 2263
2262 2264 pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
2263 2265 blocksize = (pts2write_SelfSpectra*datatypeValue)
2264 2266
2265 2267 if self.dataOut.data_cspc != None:
2266 2268 pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
2267 2269 blocksize += (pts2write_CrossSpectra*datatypeValue*2)
2268 2270
2269 2271 if self.dataOut.data_dc != None:
2270 2272 pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
2271 2273 blocksize += (pts2write_DCchannels*datatypeValue*2)
2272 2274
2273 2275 blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
2274 2276
2275 2277 return blocksize
2276 2278
2277 2279 def getDataHeader(self):
2278 2280
2279 2281 """
2280 2282 Obtiene una copia del First Header
2281 2283
2282 2284 Affected:
2283 2285 self.systemHeaderObj
2284 2286 self.radarControllerHeaderObj
2285 2287 self.dtype
2286 2288
2287 2289 Return:
2288 2290 None
2289 2291 """
2290 2292
2291 2293 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
2292 2294 self.systemHeaderObj.nChannels = self.dataOut.nChannels
2293 2295 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
2294 2296
2295 2297 self.getBasicHeader()
2296 2298
2297 2299 processingHeaderSize = 40 # bytes
2298 2300 self.processingHeaderObj.dtype = 0 # Voltage
2299 2301 self.processingHeaderObj.blockSize = self.__getBlockSize()
2300 2302 self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
2301 2303 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
2302 2304 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
2303 2305 self.processingHeaderObj.processFlags = self.__getProcessFlags()
2304 2306 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
2305 2307 self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
2306 2308 self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
2307 2309
2308 2310 if self.processingHeaderObj.totalSpectra > 0:
2309 2311 channelList = []
2310 2312 for channel in range(self.dataOut.nChannels):
2311 2313 channelList.append(channel)
2312 2314 channelList.append(channel)
2313 2315
2314 2316 pairsList = []
2315 2317 for pair in self.dataOut.pairsList:
2316 2318 pairsList.append(pair[0])
2317 2319 pairsList.append(pair[1])
2318 2320 spectraComb = channelList + pairsList
2319 2321 spectraComb = numpy.array(spectraComb,dtype="u1")
2320 2322 self.processingHeaderObj.spectraComb = spectraComb
2321 2323 sizeOfSpcComb = len(spectraComb)
2322 2324 processingHeaderSize += sizeOfSpcComb
2323 2325
2324 2326 if self.dataOut.code != None:
2325 2327 self.processingHeaderObj.code = self.dataOut.code
2326 2328 self.processingHeaderObj.nCode = self.dataOut.nCode
2327 2329 self.processingHeaderObj.nBaud = self.dataOut.nBaud
2328 2330 nCodeSize = 4 # bytes
2329 2331 nBaudSize = 4 # bytes
2330 2332 codeSize = 4 # bytes
2331 2333 sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)
2332 2334 processingHeaderSize += sizeOfCode
2333 2335
2334 2336 if self.processingHeaderObj.nWindows != 0:
2335 2337 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
2336 2338 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
2337 2339 self.processingHeaderObj.nHeights = self.dataOut.nHeights
2338 2340 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
2339 2341 sizeOfFirstHeight = 4
2340 2342 sizeOfdeltaHeight = 4
2341 2343 sizeOfnHeights = 4
2342 2344 sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows
2343 2345 processingHeaderSize += sizeOfWindows
2344 2346
2345 2347 self.processingHeaderObj.size = processingHeaderSize
2346 2348
2347 2349 class SpectraHeisWriter():
2348 2350
2349 2351 i=0
2350 2352
2351 2353 def __init__(self, dataOut):
2352 2354
2353 2355 self.wrObj = FITS()
2354 2356 self.dataOut = dataOut
2355 2357
2356 2358 def isNumber(str):
2357 2359 """
2358 2360 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
2359 2361
2360 2362 Excepciones:
2361 2363 Si un determinado string no puede ser convertido a numero
2362 2364 Input:
2363 2365 str, string al cual se le analiza para determinar si convertible a un numero o no
2364 2366
2365 2367 Return:
2366 2368 True : si el string es uno numerico
2367 2369 False : no es un string numerico
2368 2370 """
2369 2371 try:
2370 2372 float( str )
2371 2373 return True
2372 2374 except:
2373 2375 return False
2374 2376
2375 2377 def setup(self, wrpath,):
2376 2378
2377 2379 if not(os.path.exists(wrpath)):
2378 2380 os.mkdir(wrpath)
2379 2381
2380 2382 self.wrpath = wrpath
2381 2383 self.setFile = 0
2382 2384
2383 2385 def putData(self):
2384 2386 # self.wrObj.writeHeader(nChannels=self.dataOut.nChannels, nFFTPoints=self.dataOut.nFFTPoints)
2385 2387 #name = self.dataOut.utctime
2386 2388 name= time.localtime( self.dataOut.utctime)
2387 2389 ext=".fits"
2388 2390 #folder='D%4.4d%3.3d'%(name.tm_year,name.tm_yday)
2389 2391 subfolder = 'D%4.4d%3.3d' % (name.tm_year,name.tm_yday)
2390 2392
2391 2393 doypath = os.path.join( self.wrpath, subfolder )
2392 2394 if not( os.path.exists(doypath) ):
2393 2395 os.mkdir(doypath)
2394 2396 self.setFile += 1
2395 2397 file = 'D%4.4d%3.3d%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
2396 2398
2397 2399 filename = os.path.join(self.wrpath,subfolder, file)
2398 2400
2399 2401 # print self.dataOut.ippSeconds
2400 2402 freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)/(2*self.dataOut.ippSeconds)
2401 2403
2402 2404 col1=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
2403 2405 col2=self.wrObj.writeData(name="P_Ch1",format=str(self.dataOut.nFFTPoints)+'E',data=10*numpy.log10(self.dataOut.data_spc[0,:]))
2404 2406 col3=self.wrObj.writeData(name="P_Ch2",format=str(self.dataOut.nFFTPoints)+'E',data=10*numpy.log10(self.dataOut.data_spc[1,:]))
2405 2407 col4=self.wrObj.writeData(name="P_Ch3",format=str(self.dataOut.nFFTPoints)+'E',data=10*numpy.log10(self.dataOut.data_spc[2,:]))
2406 2408 col5=self.wrObj.writeData(name="P_Ch4",format=str(self.dataOut.nFFTPoints)+'E',data=10*numpy.log10(self.dataOut.data_spc[3,:]))
2407 2409 col6=self.wrObj.writeData(name="P_Ch5",format=str(self.dataOut.nFFTPoints)+'E',data=10*numpy.log10(self.dataOut.data_spc[4,:]))
2408 2410 col7=self.wrObj.writeData(name="P_Ch6",format=str(self.dataOut.nFFTPoints)+'E',data=10*numpy.log10(self.dataOut.data_spc[5,:]))
2409 2411 col8=self.wrObj.writeData(name="P_Ch7",format=str(self.dataOut.nFFTPoints)+'E',data=10*numpy.log10(self.dataOut.data_spc[6,:]))
2410 2412 col9=self.wrObj.writeData(name="P_Ch8",format=str(self.dataOut.nFFTPoints)+'E',data=10*numpy.log10(self.dataOut.data_spc[7,:]))
2411 2413 #n=numpy.arange((100))
2412 2414 n=self.dataOut.data_spc[6,:]
2413 2415 a=self.wrObj.cFImage(n)
2414 2416 b=self.wrObj.Ctable(col1,col2,col3,col4,col5,col6,col7,col8,col9)
2415 2417 self.wrObj.CFile(a,b)
2416 2418 self.wrObj.wFile(filename)
2417 2419 return 1
2418 2420
2419 2421 class FITS:
2420 2422
2421 2423 name=None
2422 2424 format=None
2423 2425 array =None
2424 2426 data =None
2425 2427 thdulist=None
2426 2428
2427 2429 def __init__(self):
2428 2430
2429 2431 pass
2430 2432
2431 2433 def setColF(self,name,format,array):
2432 2434 self.name=name
2433 2435 self.format=format
2434 2436 self.array=array
2435 2437 a1=numpy.array([self.array],dtype=numpy.float32)
2436 2438 self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)
2437 2439 return self.col1
2438 2440
2439 2441 # def setColP(self,name,format,data):
2440 2442 # self.name=name
2441 2443 # self.format=format
2442 2444 # self.data=data
2443 2445 # a2=numpy.array([self.data],dtype=numpy.float32)
2444 2446 # self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
2445 2447 # return self.col2
2446 2448
2447 2449 def writeHeader(self,):
2448 2450 pass
2449 2451
2450 2452 def writeData(self,name,format,data):
2451 2453 self.name=name
2452 2454 self.format=format
2453 2455 self.data=data
2454 2456 a2=numpy.array([self.data],dtype=numpy.float32)
2455 2457 self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
2456 2458 return self.col2
2457 2459
2458 2460 def cFImage(self,n):
2459 2461 self.hdu= pyfits.PrimaryHDU(n)
2460 2462 return self.hdu
2461 2463
2462 2464 def Ctable(self,col1,col2,col3,col4,col5,col6,col7,col8,col9):
2463 2465 self.cols=pyfits.ColDefs( [col1,col2,col3,col4,col5,col6,col7,col8,col9])
2464 2466 self.tbhdu = pyfits.new_table(self.cols)
2465 2467 return self.tbhdu
2466 2468
2467 2469 def CFile(self,hdu,tbhdu):
2468 2470 self.thdulist=pyfits.HDUList([hdu,tbhdu])
2469 2471
2470 2472 def wFile(self,filename):
2471 2473 self.thdulist.writeto(filename) No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,439 +1,435
1 1 '''
2 2
3 3 $Author: dsuarez $
4 4 $Id: Processor.py 1 2012-11-12 18:56:07Z dsuarez $
5 5 '''
6 6 import os
7 7 import numpy
8 8 import datetime
9 9 import time
10 10
11 11 from jrodata import *
12 12 from jrodataIO import *
13 13 from jroplot import *
14 14
15 15 class ProcessingUnit:
16 16
17 17 """
18 18 Esta es la clase base para el procesamiento de datos.
19 19
20 20 Contiene el metodo "call" para llamar operaciones. Las operaciones pueden ser:
21 21 - Metodos internos (callMethod)
22 22 - Objetos del tipo Operation (callObject). Antes de ser llamados, estos objetos
23 23 tienen que ser agreagados con el metodo "add".
24 24
25 25 """
26 26 # objeto de datos de entrada (Voltage, Spectra o Correlation)
27 27 dataIn = None
28 28
29 29 # objeto de datos de entrada (Voltage, Spectra o Correlation)
30 30 dataOut = None
31 31
32 32
33 33 objectDict = None
34 34
35 35 def __init__(self):
36 36
37 37 self.objectDict = {}
38 38
39 39 def addOperation(self, object, objId):
40 40
41 41 """
42 42 Agrega el objeto "object" a la lista de objetos "self.objectList" y retorna el
43 43 identificador asociado a este objeto.
44 44
45 45 Input:
46 46
47 47 object : objeto de la clase "Operation"
48 48
49 49 Return:
50 50
51 51 objId : identificador del objeto, necesario para ejecutar la operacion
52 52 """
53 53
54 54 self.objectDict[objId] = object
55 55
56 56 return objId
57 57
58 58 def operation(self, **kwargs):
59 59
60 60 """
61 61 Operacion directa sobre la data (dataout.data). Es necesario actualizar los valores de los
62 62 atributos del objeto dataOut
63 63
64 64 Input:
65 65
66 66 **kwargs : Diccionario de argumentos de la funcion a ejecutar
67 67 """
68 68
69 69 raise ValueError, "ImplementedError"
70 70
71 71 def callMethod(self, name, **kwargs):
72 72
73 73 """
74 74 Ejecuta el metodo con el nombre "name" y con argumentos **kwargs de la propia clase.
75 75
76 76 Input:
77 77 name : nombre del metodo a ejecutar
78 78
79 79 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
80 80
81 81 """
82 82
83 83 methodToCall = getattr(self, name)
84 84
85 85 methodToCall(**kwargs)
86 86
87 87 def callObject(self, objId, **kwargs):
88 88
89 89 """
90 90 Ejecuta la operacion asociada al identificador del objeto "objId"
91 91
92 92 Input:
93 93
94 94 objId : identificador del objeto a ejecutar
95 95
96 96 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
97 97
98 98 Return:
99 99
100 100 None
101 101 """
102 102
103 103 object = self.objectDict[objId]
104 104
105 105 object.run(self.dataOut, **kwargs)
106 106
107 107 def call(self, operationConf, **kwargs):
108 108
109 109 """
110 110 Ejecuta la operacion "operationConf.name" con los argumentos "**kwargs". La operacion puede
111 111 ser de dos tipos:
112 112
113 113 1. Un metodo propio de esta clase:
114 114
115 115 operation.type = "self"
116 116
117 117 2. El metodo "run" de un objeto del tipo Operation o de un derivado de ella:
118 118 operation.type = "other".
119 119
120 120 Este objeto de tipo Operation debe de haber sido agregado antes con el metodo:
121 121 "addOperation" e identificado con el operation.id
122 122
123 123
124 124 con el id de la operacion.
125 125
126 126 Input:
127 127
128 128 Operation : Objeto del tipo operacion con los atributos: name, type y id.
129 129
130 130 """
131 131 if self.dataIn.isEmpty():
132 132 return None
133 133
134 134 if operationConf.type == 'self':
135 135 self.callMethod(operationConf.name, **kwargs)
136 136 return
137 137
138 138 if operationConf.type == 'other':
139 139 self.callObject(operationConf.id, **kwargs)
140 140 return
141 141
142 def setInput(self, dataIn):
143
144 self.dataIn = dataIn
145
146 def getOutput(self):
147
148 return self.dataOut
149
142 150 class Operation():
143 151
144 152 """
145 153 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
146 154 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
147 155 acumulacion dentro de esta clase
148 156
149 157 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
150 158
151 159 """
152 160
153 161 __buffer = None
154 162 __isConfig = False
155 163
156 164 def __init__(self):
157 165
158 166 pass
159 167
160 168 def run(self, dataIn, **kwargs):
161 169
162 170 """
163 171 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los atributos del objeto dataIn.
164 172
165 173 Input:
166 174
167 175 dataIn : objeto del tipo JROData
168 176
169 177 Return:
170 178
171 179 None
172 180
173 181 Affected:
174 182 __buffer : buffer de recepcion de datos.
175 183
176 184 """
177 185
178 186 raise ValueError, "ImplementedError"
179 187
180 188 class VoltageProc(ProcessingUnit):
181 189
182 190
183 191 def __init__(self):
184 192
185 193 self.objectDict = {}
186 194 self.dataOut = Voltage()
187 195
188
189 def setup(self, dataIn=None, dataOut=None):
190
191 self.dataIn = dataIn
192
193 if self.dataOut == None:
194 dataOut = Voltage()
195
196 self.dataOut = dataOut
197
198 return self.dataOut
199
200 196 def init(self):
201 197
202 198 self.dataOut.copy(self.dataIn)
203 199 # No necesita copiar en cada init() los atributos de dataIn
204 200 # la copia deberia hacerse por cada nuevo bloque de datos
205 201
206 202 def selectChannels(self, channelList):
207 203
208 204 if self.dataIn.isEmpty():
209 205 return 0
210 206
211 207 self.selectChannelsByIndex(channelList)
212 208
213 209 def selectChannelsByIndex(self, channelIndexList):
214 210 """
215 211 Selecciona un bloque de datos en base a canales segun el channelIndexList
216 212
217 213 Input:
218 214 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
219 215
220 216 Affected:
221 217 self.dataOut.data
222 218 self.dataOut.channelIndexList
223 219 self.dataOut.nChannels
224 220 self.dataOut.m_ProcessingHeader.totalSpectra
225 221 self.dataOut.systemHeaderObj.numChannels
226 222 self.dataOut.m_ProcessingHeader.blockSize
227 223
228 224 Return:
229 225 None
230 226 """
231 227
232 228 for channel in channelIndexList:
233 229 if channel not in self.dataOut.channelIndexList:
234 230 raise ValueError, "The value %d in channelIndexList is not valid" %channel
235 231
236 232 nChannels = len(channelIndexList)
237 233
238 234 data = self.dataOut.data[channelIndexList,:]
239 235
240 236 self.dataOut.data = data
241 237 self.dataOut.channelIndexList = channelIndexList
242 238 self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
243 239 self.dataOut.nChannels = nChannels
244 240
245 241 return 1
246 242
247 243 class CohInt(Operation):
248 244
249 245 __profIndex = 0
250 246 __withOverapping = False
251 247
252 248 __byTime = False
253 249 __initime = None
254 250 __lastdatatime = None
255 251 __integrationtime = None
256 252
257 253 __buffer = None
258 254
259 255 __dataReady = False
260 256
261 257 nCohInt = None
262 258
263 259
264 260 def __init__(self):
265 261
266 262 self.__isConfig = False
267 263
268 264 def setup(self, nCohInt=None, timeInterval=None, overlapping=False):
269 265 """
270 266 Set the parameters of the integration class.
271 267
272 268 Inputs:
273 269
274 270 nCohInt : Number of coherent integrations
275 271 timeInterval : Time of integration. If the parameter "nCohInt" is selected this one does not work
276 272 overlapping :
277 273
278 274 """
279 275
280 276 self.__initime = None
281 277 self.__lastdatatime = 0
282 278 self.__buffer = None
283 279 self.__dataReady = False
284 280
285 281
286 282 if nCohInt == None and timeInterval == None:
287 283 raise ValueError, "nCohInt or timeInterval should be specified ..."
288 284
289 285 if nCohInt != None:
290 286 self.nCohInt = nCohInt
291 287 self.__byTime = False
292 288 else:
293 289 self.__integrationtime = timeInterval * 60. #if (type(timeInterval)!=integer) -> change this line
294 290 self.nCohInt = 9999
295 291 self.__byTime = True
296 292
297 293 if overlapping:
298 294 self.__withOverapping = True
299 295 self.__buffer = None
300 296 else:
301 297 self.__withOverapping = False
302 298 self.__buffer = 0
303 299
304 300 self.__profIndex = 0
305 301
306 302 def putData(self, data):
307 303
308 304 """
309 305 Add a profile to the __buffer and increase in one the __profileIndex
310 306
311 307 """
312 308
313 309 if not self.__withOverapping:
314 310 self.__buffer += data
315 311 self.__profIndex += 1
316 312 return
317 313
318 314 #Overlapping data
319 315 nChannels, nHeis = data.shape
320 316 data = numpy.reshape(data, (1, nChannels, nHeis))
321 317
322 318 #If the buffer is empty then it takes the data value
323 319 if self.__buffer == None:
324 320 self.__buffer = data
325 321 self.__profIndex += 1
326 322 return
327 323
328 324 #If the buffer length is lower than nCohInt then stakcing the data value
329 325 if self.__profIndex < self.nCohInt:
330 326 self.__buffer = numpy.vstack((self.__buffer, data))
331 327 self.__profIndex += 1
332 328 return
333 329
334 330 #If the buffer length is equal to nCohInt then replacing the last buffer value with the data value
335 331 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
336 332 self.__buffer[self.nCohInt-1] = data
337 333 self.__profIndex = self.nCohInt
338 334 return
339 335
340 336
341 337 def pushData(self):
342 338 """
343 339 Return the sum of the last profiles and the profiles used in the sum.
344 340
345 341 Affected:
346 342
347 343 self.__profileIndex
348 344
349 345 """
350 346
351 347 if not self.__withOverapping:
352 348 data = self.__buffer
353 349 nCohInt = self.__profIndex
354 350
355 351 self.__buffer = 0
356 352 self.__profIndex = 0
357 353
358 354 return data, nCohInt
359 355
360 356 #Integration with Overlapping
361 357 data = numpy.sum(self.__buffer, axis=0)
362 358 nCohInt = self.__profIndex
363 359
364 360 return data, nCohInt
365 361
366 362 def byProfiles(self, data):
367 363
368 364 self.__dataReady = False
369 365 avgdata = None
370 366 nCohInt = None
371 367
372 368 self.putData(data)
373 369
374 370 if self.__profIndex == self.nCohInt:
375 371
376 372 avgdata, nCohInt = self.pushData()
377 373 self.__dataReady = True
378 374
379 375 return avgdata
380 376
381 377 def byTime(self, data, datatime):
382 378
383 379 self.__dataReady = False
384 380 avgdata = None
385 381 nCohInt = None
386 382
387 383 self.putData(data)
388 384
389 385 if (datatime - self.__initime) >= self.__integrationtime:
390 386 avgdata, nCohInt = self.pushData()
391 387 self.nCohInt = nCohInt
392 388 self.__dataReady = True
393 389
394 390 return avgdata
395 391
396 392 def integrate(self, data, datatime=None):
397 393
398 394 if self.__initime == None:
399 395 self.__initime = datatime
400 396
401 397 if self.__byTime:
402 398 avgdata = self.byTime(data, datatime)
403 399 else:
404 400 avgdata = self.byProfiles(data)
405 401
406 402
407 403 self.__lastdatatime = datatime
408 404
409 405 if avgdata == None:
410 406 return None, None
411 407
412 408 avgdatatime = self.__initime
413 409
414 410 deltatime = datatime -self.__lastdatatime
415 411
416 412 if not self.__withOverapping:
417 413 self.__initime = datatime
418 414 else:
419 415 self.__initime += deltatime
420 416
421 417 return avgdata, avgdatatime
422 418
423 419 def run(self, dataOut, nCohInt=None, timeInterval=None, overlapping=False):
424 420
425 421 if not self.__isConfig:
426 422 self.setup(nCohInt, timeInterval, overlapping)
427 423 self.__isConfig = True
428 424
429 425 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
430 426
431 427 # dataOut.timeInterval *= nCohInt
432 428 dataOut.flagNoData = True
433 429
434 430 if self.__dataReady:
435 431 dataOut.data = avgdata
436 432 dataOut.timeInterval *= self.nCohInt
437 433 dataOut.nCohInt *= self.nCohInt
438 434 dataOut.utctime = avgdatatime
439 435 dataOut.flagNoData = False No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now