##// END OF EJS Templates
schain
RSS
GUI: Bug fixed when *Read from header* is selected...
Miguel Valdez -
r655:9d147099e1e8
parent child
Show More

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,1596 +1,1596
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 23
24 24 LOCALTIME = True
25 25
26 26 def isNumber(cad):
27 27 """
28 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 29
30 30 Excepciones:
31 31 Si un determinado string no puede ser convertido a numero
32 32 Input:
33 33 str, string al cual se le analiza para determinar si convertible a un numero o no
34 34
35 35 Return:
36 36 True : si el string es uno numerico
37 37 False : no es un string numerico
38 38 """
39 39 try:
40 40 float( cad )
41 41 return True
42 42 except:
43 43 return False
44 44
45 45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 48
49 49 Inputs:
50 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 51
52 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 53 segundos contados desde 01/01/1970.
54 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 55 segundos contados desde 01/01/1970.
56 56
57 57 Return:
58 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 59 fecha especificado, de lo contrario retorna False.
60 60
61 61 Excepciones:
62 62 Si el archivo no existe o no puede ser abierto
63 63 Si la cabecera no puede ser leida.
64 64
65 65 """
66 66 basicHeaderObj = BasicHeader(LOCALTIME)
67 67
68 68 try:
69 69 fp = open(filename,'rb')
70 70 except IOError:
71 71 traceback.print_exc()
72 72 raise IOError, "The file %s can't be opened" %(filename)
73 73
74 74 sts = basicHeaderObj.read(fp)
75 75 fp.close()
76 76
77 77 if not(sts):
78 78 print "Skipping the file %s because it has not a valid header" %(filename)
79 79 return 0
80 80
81 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 82 return 0
83 83
84 84 return 1
85 85
86 86 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
87 87 """
88 88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89 89
90 90 Inputs:
91 91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92 92
93 93 startDate : fecha inicial del rango seleccionado en formato datetime.date
94 94
95 95 endDate : fecha final del rango seleccionado en formato datetime.date
96 96
97 97 startTime : tiempo inicial del rango seleccionado en formato datetime.time
98 98
99 99 endTime : tiempo final del rango seleccionado en formato datetime.time
100 100
101 101 Return:
102 102 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
103 103 fecha especificado, de lo contrario retorna False.
104 104
105 105 Excepciones:
106 106 Si el archivo no existe o no puede ser abierto
107 107 Si la cabecera no puede ser leida.
108 108
109 109 """
110 110
111 111
112 112 try:
113 113 fp = open(filename,'rb')
114 114 except IOError:
115 115 traceback.print_exc()
116 116 raise IOError, "The file %s can't be opened" %(filename)
117 117
118 118 basicHeaderObj = BasicHeader(LOCALTIME)
119 119 sts = basicHeaderObj.read(fp)
120 120 fp.close()
121 121
122 122 thisDatetime = basicHeaderObj.datatime
123 123 thisDate = thisDatetime.date()
124 124 thisTime = thisDatetime.time()
125 125
126 126 if not(sts):
127 127 print "Skipping the file %s because it has not a valid header" %(filename)
128 128 return None
129 129
130 130 #General case
131 131 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
132 132 #-----------o----------------------------o-----------
133 133 # startTime endTime
134 134
135 135 if endTime >= startTime:
136 136 if (thisTime < startTime) or (thisTime > endTime):
137 137 return None
138 138
139 139 return thisDatetime
140 140
141 141 #If endTime < startTime then endTime belongs to the next day
142 142
143 143
144 144 #<<<<<<<<<<<o o>>>>>>>>>>>
145 145 #-----------o----------------------------o-----------
146 146 # endTime startTime
147 147
148 148 if (thisDate == startDate) and (thisTime < startTime):
149 149 return None
150 150
151 151 if (thisDate == endDate) and (thisTime > endTime):
152 152 return None
153 153
154 154 if (thisTime < startTime) and (thisTime > endTime):
155 155 return None
156 156
157 157 return thisDatetime
158 158
159 159 def isFolderInDateRange(folder, startDate=None, endDate=None):
160 160 """
161 161 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
162 162
163 163 Inputs:
164 164 folder : nombre completo del directorio.
165 165 Su formato deberia ser "/path_root/?YYYYDDD"
166 166
167 167 siendo:
168 168 YYYY : Anio (ejemplo 2015)
169 169 DDD : Dia del anio (ejemplo 305)
170 170
171 171 startDate : fecha inicial del rango seleccionado en formato datetime.date
172 172
173 173 endDate : fecha final del rango seleccionado en formato datetime.date
174 174
175 175 Return:
176 176 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
177 177 fecha especificado, de lo contrario retorna False.
178 178 Excepciones:
179 179 Si el directorio no tiene el formato adecuado
180 180 """
181 181
182 182 basename = os.path.basename(folder)
183 183
184 184 if not isRadarFolder(basename):
185 185 raise IOError, "The folder %s has not the rigth format" %folder
186 186
187 187 if startDate and endDate:
188 188 thisDate = getDateFromRadarFolder(basename)
189 189
190 190 if thisDate < startDate:
191 191 return 0
192 192
193 193 if thisDate > endDate:
194 194 return 0
195 195
196 196 return 1
197 197
198 198 def isFileInDateRange(filename, startDate=None, endDate=None):
199 199 """
200 200 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
201 201
202 202 Inputs:
203 203 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
204 204
205 205 Su formato deberia ser "?YYYYDDDsss"
206 206
207 207 siendo:
208 208 YYYY : Anio (ejemplo 2015)
209 209 DDD : Dia del anio (ejemplo 305)
210 210 sss : set
211 211
212 212 startDate : fecha inicial del rango seleccionado en formato datetime.date
213 213
214 214 endDate : fecha final del rango seleccionado en formato datetime.date
215 215
216 216 Return:
217 217 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
218 218 fecha especificado, de lo contrario retorna False.
219 219 Excepciones:
220 220 Si el archivo no tiene el formato adecuado
221 221 """
222 222
223 223 basename = os.path.basename(filename)
224 224
225 225 if not isRadarFile(basename):
226 226 raise IOError, "The filename %s has not the rigth format" %filename
227 227
228 228 if startDate and endDate:
229 229 thisDate = getDateFromRadarFile(basename)
230 230
231 231 if thisDate < startDate:
232 232 return 0
233 233
234 234 if thisDate > endDate:
235 235 return 0
236 236
237 237 return 1
238 238
239 239 def getFileFromSet(path, ext, set):
240 240 validFilelist = []
241 241 fileList = os.listdir(path)
242 242
243 243 # 0 1234 567 89A BCDE
244 244 # H YYYY DDD SSS .ext
245 245
246 246 for thisFile in fileList:
247 247 try:
248 248 year = int(thisFile[1:5])
249 249 doy = int(thisFile[5:8])
250 250 except:
251 251 continue
252 252
253 253 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
254 254 continue
255 255
256 256 validFilelist.append(thisFile)
257 257
258 258 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
259 259
260 260 if len(myfile)!= 0:
261 261 return myfile[0]
262 262 else:
263 263 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
264 264 print 'the filename %s does not exist'%filename
265 265 print '...going to the last file: '
266 266
267 267 if validFilelist:
268 268 validFilelist = sorted( validFilelist, key=str.lower )
269 269 return validFilelist[-1]
270 270
271 271 return None
272 272
273 273 def getlastFileFromPath(path, ext):
274 274 """
275 275 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
276 276 al final de la depuracion devuelve el ultimo file de la lista que quedo.
277 277
278 278 Input:
279 279 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
280 280 ext : extension de los files contenidos en una carpeta
281 281
282 282 Return:
283 283 El ultimo file de una determinada carpeta, no se considera el path.
284 284 """
285 285 validFilelist = []
286 286 fileList = os.listdir(path)
287 287
288 288 # 0 1234 567 89A BCDE
289 289 # H YYYY DDD SSS .ext
290 290
291 291 for thisFile in fileList:
292 292
293 293 year = thisFile[1:5]
294 294 if not isNumber(year):
295 295 continue
296 296
297 297 doy = thisFile[5:8]
298 298 if not isNumber(doy):
299 299 continue
300 300
301 301 year = int(year)
302 302 doy = int(doy)
303 303
304 304 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
305 305 continue
306 306
307 307 validFilelist.append(thisFile)
308 308
309 309 if validFilelist:
310 310 validFilelist = sorted( validFilelist, key=str.lower )
311 311 return validFilelist[-1]
312 312
313 313 return None
314 314
315 315 def checkForRealPath(path, foldercounter, year, doy, set, ext):
316 316 """
317 317 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
318 318 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
319 319 el path exacto de un determinado file.
320 320
321 321 Example :
322 322 nombre correcto del file es .../.../D2009307/P2009307367.ext
323 323
324 324 Entonces la funcion prueba con las siguientes combinaciones
325 325 .../.../y2009307367.ext
326 326 .../.../Y2009307367.ext
327 327 .../.../x2009307/y2009307367.ext
328 328 .../.../x2009307/Y2009307367.ext
329 329 .../.../X2009307/y2009307367.ext
330 330 .../.../X2009307/Y2009307367.ext
331 331 siendo para este caso, la ultima combinacion de letras, identica al file buscado
332 332
333 333 Return:
334 334 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
335 335 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
336 336 para el filename
337 337 """
338 338 fullfilename = None
339 339 find_flag = False
340 340 filename = None
341 341
342 342 prefixDirList = [None,'d','D']
343 343 if ext.lower() == ".r": #voltage
344 344 prefixFileList = ['d','D']
345 345 elif ext.lower() == ".pdata": #spectra
346 346 prefixFileList = ['p','P']
347 347 else:
348 348 return None, filename
349 349
350 350 #barrido por las combinaciones posibles
351 351 for prefixDir in prefixDirList:
352 352 thispath = path
353 353 if prefixDir != None:
354 354 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
355 355 if foldercounter == 0:
356 356 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
357 357 else:
358 358 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
359 359 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
360 360 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
361 361 fullfilename = os.path.join( thispath, filename ) #formo el path completo
362 362
363 363 if os.path.exists( fullfilename ): #verifico que exista
364 364 find_flag = True
365 365 break
366 366 if find_flag:
367 367 break
368 368
369 369 if not(find_flag):
370 370 return None, filename
371 371
372 372 return fullfilename, filename
373 373
374 374 def isRadarFolder(folder):
375 375 try:
376 376 year = int(folder[1:5])
377 377 doy = int(folder[5:8])
378 378 except:
379 379 return 0
380 380
381 381 return 1
382 382
383 383 def isRadarFile(file):
384 384 try:
385 385 year = int(file[1:5])
386 386 doy = int(file[5:8])
387 387 set = int(file[8:11])
388 388 except:
389 389 return 0
390 390
391 391 return 1
392 392
393 393 def getDateFromRadarFile(file):
394 394 try:
395 395 year = int(file[1:5])
396 396 doy = int(file[5:8])
397 397 set = int(file[8:11])
398 398 except:
399 399 return None
400 400
401 401 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
402 402 return thisDate
403 403
404 404 def getDateFromRadarFolder(folder):
405 405 try:
406 406 year = int(folder[1:5])
407 407 doy = int(folder[5:8])
408 408 except:
409 409 return None
410 410
411 411 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
412 412 return thisDate
413 413
414 414 class JRODataIO:
415 415
416 416 c = 3E8
417 417
418 418 isConfig = False
419 419
420 420 basicHeaderObj = None
421 421
422 422 systemHeaderObj = None
423 423
424 424 radarControllerHeaderObj = None
425 425
426 426 processingHeaderObj = None
427 427
428 428 online = 0
429 429
430 430 dtype = None
431 431
432 432 pathList = []
433 433
434 434 filenameList = []
435 435
436 436 filename = None
437 437
438 438 ext = None
439 439
440 440 flagIsNewFile = 1
441 441
442 442 flagDiscontinuousBlock = 0
443 443
444 444 flagIsNewBlock = 0
445 445
446 446 fp = None
447 447
448 448 firstHeaderSize = 0
449 449
450 450 basicHeaderSize = 24
451 451
452 452 versionFile = 1103
453 453
454 454 fileSize = None
455 455
456 456 # ippSeconds = None
457 457
458 458 fileSizeByHeader = None
459 459
460 460 fileIndex = None
461 461
462 462 profileIndex = None
463 463
464 464 blockIndex = None
465 465
466 466 nTotalBlocks = None
467 467
468 468 maxTimeStep = 30
469 469
470 470 lastUTTime = None
471 471
472 472 datablock = None
473 473
474 474 dataOut = None
475 475
476 476 blocksize = None
477 477
478 478 getByBlock = False
479 479
480 480 def __init__(self):
481 481
482 482 raise ValueError, "Not implemented"
483 483
484 484 def run(self):
485 485
486 486 raise ValueError, "Not implemented"
487 487
488 488 def getDtypeWidth(self):
489 489
490 490 dtype_index = get_dtype_index(self.dtype)
491 491 dtype_width = get_dtype_width(dtype_index)
492 492
493 493 return dtype_width
494 494
495 495 class JRODataReader(JRODataIO):
496 496
497 497 nReadBlocks = 0
498 498
499 499 delay = 10 #number of seconds waiting a new file
500 500
501 501 nTries = 3 #quantity tries
502 502
503 503 nFiles = 3 #number of files for searching
504 504
505 505 path = None
506 506
507 507 foldercounter = 0
508 508
509 509 flagNoMoreFiles = 0
510 510
511 511 datetimeList = []
512 512
513 513 __isFirstTimeOnline = 1
514 514
515 515 __printInfo = True
516 516
517 517 profileIndex = None
518 518
519 519 nTxs = 1
520 520
521 521 txIndex = None
522 522
523 523 def __init__(self):
524 524
525 525 """
526 526
527 527 """
528 528
529 529 # raise NotImplementedError, "This method has not been implemented"
530 530
531 531
532 532 def createObjByDefault(self):
533 533 """
534 534
535 535 """
536 536 raise NotImplementedError, "This method has not been implemented"
537 537
538 538 def getBlockDimension(self):
539 539
540 540 raise NotImplementedError, "No implemented"
541 541
542 542 def __searchFilesOffLine(self,
543 543 path,
544 544 startDate=None,
545 545 endDate=None,
546 546 startTime=datetime.time(0,0,0),
547 547 endTime=datetime.time(23,59,59),
548 548 set=None,
549 549 expLabel='',
550 550 ext='.r',
551 551 walk=True):
552 552
553 553 self.filenameList = []
554 554 self.datetimeList = []
555 555
556 556 pathList = []
557 557
558 558 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
559 559
560 560 if dateList == []:
561 561 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
562 562 datetime.datetime.combine(startDate,startTime).ctime(),
563 563 datetime.datetime.combine(endDate,endTime).ctime())
564 564
565 565 return None, None
566 566
567 567 if len(dateList) > 1:
568 568 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
569 569 else:
570 570 print "[Reading] data was found for the date %s" %(dateList[0])
571 571
572 572 filenameList = []
573 573 datetimeList = []
574 574
575 575 for thisPath in pathList:
576 576 # thisPath = pathList[pathDict[file]]
577 577
578 578 fileList = glob.glob1(thisPath, "*%s" %ext)
579 579 fileList.sort()
580 580
581 581 for file in fileList:
582 582
583 583 filename = os.path.join(thisPath,file)
584 584
585 585 if not isFileInDateRange(filename, startDate, endDate):
586 586 continue
587 587
588 588 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
589 589
590 590 if not(thisDatetime):
591 591 continue
592 592
593 593 filenameList.append(filename)
594 594 datetimeList.append(thisDatetime)
595 595
596 596 if not(filenameList):
597 print "[Reading] Any file was found int time range %s - %s" %(startTime.ctime(), endTime.ctime())
597 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
598 598 return None, None
599 599
600 600 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
601 601 print
602 602
603 603 for i in range(len(filenameList)):
604 604 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
605 605
606 606 self.filenameList = filenameList
607 607 self.datetimeList = datetimeList
608 608
609 609 return pathList, filenameList
610 610
611 611 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
612 612
613 613 """
614 614 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
615 615 devuelve el archivo encontrado ademas de otros datos.
616 616
617 617 Input:
618 618 path : carpeta donde estan contenidos los files que contiene data
619 619
620 620 expLabel : Nombre del subexperimento (subfolder)
621 621
622 622 ext : extension de los files
623 623
624 624 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
625 625
626 626 Return:
627 627 directory : eL directorio donde esta el file encontrado
628 628 filename : el ultimo file de una determinada carpeta
629 629 year : el anho
630 630 doy : el numero de dia del anho
631 631 set : el set del archivo
632 632
633 633
634 634 """
635 635 dirList = []
636 636
637 637 if not walk:
638 638 fullpath = path
639 639 foldercounter = 0
640 640 else:
641 641 #Filtra solo los directorios
642 642 for thisPath in os.listdir(path):
643 643 if not os.path.isdir(os.path.join(path,thisPath)):
644 644 continue
645 645 if not isRadarFolder(thisPath):
646 646 continue
647 647
648 648 dirList.append(thisPath)
649 649
650 650 if not(dirList):
651 651 return None, None, None, None, None, None
652 652
653 653 dirList = sorted( dirList, key=str.lower )
654 654
655 655 doypath = dirList[-1]
656 656 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
657 657 fullpath = os.path.join(path, doypath, expLabel)
658 658
659 659
660 660 print "[Reading] %s folder was found: " %(fullpath )
661 661
662 662 if set == None:
663 663 filename = getlastFileFromPath(fullpath, ext)
664 664 else:
665 665 filename = getFileFromSet(fullpath, ext, set)
666 666
667 667 if not(filename):
668 668 return None, None, None, None, None, None
669 669
670 670 print "[Reading] %s file was found" %(filename)
671 671
672 672 if not(self.__verifyFile(os.path.join(fullpath, filename))):
673 673 return None, None, None, None, None, None
674 674
675 675 year = int( filename[1:5] )
676 676 doy = int( filename[5:8] )
677 677 set = int( filename[8:11] )
678 678
679 679 return fullpath, foldercounter, filename, year, doy, set
680 680
681 681 def __setNextFileOffline(self):
682 682
683 683 idFile = self.fileIndex
684 684
685 685 while (True):
686 686 idFile += 1
687 687 if not(idFile < len(self.filenameList)):
688 688 self.flagNoMoreFiles = 1
689 689 # print "[Reading] No more Files"
690 690 return 0
691 691
692 692 filename = self.filenameList[idFile]
693 693
694 694 if not(self.__verifyFile(filename)):
695 695 continue
696 696
697 697 fileSize = os.path.getsize(filename)
698 698 fp = open(filename,'rb')
699 699 break
700 700
701 701 self.flagIsNewFile = 1
702 702 self.fileIndex = idFile
703 703 self.filename = filename
704 704 self.fileSize = fileSize
705 705 self.fp = fp
706 706
707 707 # print "[Reading] Setting the file: %s"%self.filename
708 708
709 709 return 1
710 710
711 711 def __setNextFileOnline(self):
712 712 """
713 713 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
714 714 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
715 715 siguientes.
716 716
717 717 Affected:
718 718 self.flagIsNewFile
719 719 self.filename
720 720 self.fileSize
721 721 self.fp
722 722 self.set
723 723 self.flagNoMoreFiles
724 724
725 725 Return:
726 726 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
727 727 1 : si el file fue abierto con exito y esta listo a ser leido
728 728
729 729 Excepciones:
730 730 Si un determinado file no puede ser abierto
731 731 """
732 732 nFiles = 0
733 733 fileOk_flag = False
734 734 firstTime_flag = True
735 735
736 736 self.set += 1
737 737
738 738 if self.set > 999:
739 739 self.set = 0
740 740 self.foldercounter += 1
741 741
742 742 #busca el 1er file disponible
743 743 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
744 744 if fullfilename:
745 745 if self.__verifyFile(fullfilename, False):
746 746 fileOk_flag = True
747 747
748 748 #si no encuentra un file entonces espera y vuelve a buscar
749 749 if not(fileOk_flag):
750 750 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
751 751
752 752 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
753 753 tries = self.nTries
754 754 else:
755 755 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
756 756
757 757 for nTries in range( tries ):
758 758 if firstTime_flag:
759 759 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
760 760 sleep( self.delay )
761 761 else:
762 762 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
763 763
764 764 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
765 765 if fullfilename:
766 766 if self.__verifyFile(fullfilename):
767 767 fileOk_flag = True
768 768 break
769 769
770 770 if fileOk_flag:
771 771 break
772 772
773 773 firstTime_flag = False
774 774
775 775 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
776 776 self.set += 1
777 777
778 778 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
779 779 self.set = 0
780 780 self.doy += 1
781 781 self.foldercounter = 0
782 782
783 783 if fileOk_flag:
784 784 self.fileSize = os.path.getsize( fullfilename )
785 785 self.filename = fullfilename
786 786 self.flagIsNewFile = 1
787 787 if self.fp != None: self.fp.close()
788 788 self.fp = open(fullfilename, 'rb')
789 789 self.flagNoMoreFiles = 0
790 790 # print '[Reading] Setting the file: %s' % fullfilename
791 791 else:
792 792 self.fileSize = 0
793 793 self.filename = None
794 794 self.flagIsNewFile = 0
795 795 self.fp = None
796 796 self.flagNoMoreFiles = 1
797 797 # print '[Reading] No more files to read'
798 798
799 799 return fileOk_flag
800 800
801 801 def setNextFile(self):
802 802 if self.fp != None:
803 803 self.fp.close()
804 804
805 805 if self.online:
806 806 newFile = self.__setNextFileOnline()
807 807 else:
808 808 newFile = self.__setNextFileOffline()
809 809
810 810 if not(newFile):
811 811 print '[Reading] No more files to read'
812 812 return 0
813 813
814 814 print '[Reading] Setting the file: %s' % self.filename
815 815
816 816 self.__readFirstHeader()
817 817 self.nReadBlocks = 0
818 818 return 1
819 819
820 820 def __waitNewBlock(self):
821 821 """
822 822 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
823 823
824 824 Si el modo de lectura es OffLine siempre retorn 0
825 825 """
826 826 if not self.online:
827 827 return 0
828 828
829 829 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
830 830 return 0
831 831
832 832 currentPointer = self.fp.tell()
833 833
834 834 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
835 835
836 836 for nTries in range( self.nTries ):
837 837
838 838 self.fp.close()
839 839 self.fp = open( self.filename, 'rb' )
840 840 self.fp.seek( currentPointer )
841 841
842 842 self.fileSize = os.path.getsize( self.filename )
843 843 currentSize = self.fileSize - currentPointer
844 844
845 845 if ( currentSize >= neededSize ):
846 846 self.basicHeaderObj.read(self.fp)
847 847 return 1
848 848
849 849 if self.fileSize == self.fileSizeByHeader:
850 850 # self.flagEoF = True
851 851 return 0
852 852
853 853 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
854 854 sleep( self.delay )
855 855
856 856
857 857 return 0
858 858
859 859 def waitDataBlock(self,pointer_location):
860 860
861 861 currentPointer = pointer_location
862 862
863 863 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
864 864
865 865 for nTries in range( self.nTries ):
866 866 self.fp.close()
867 867 self.fp = open( self.filename, 'rb' )
868 868 self.fp.seek( currentPointer )
869 869
870 870 self.fileSize = os.path.getsize( self.filename )
871 871 currentSize = self.fileSize - currentPointer
872 872
873 873 if ( currentSize >= neededSize ):
874 874 return 1
875 875
876 876 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
877 877 sleep( self.delay )
878 878
879 879 return 0
880 880
881 881 def __jumpToLastBlock(self):
882 882
883 883 if not(self.__isFirstTimeOnline):
884 884 return
885 885
886 886 csize = self.fileSize - self.fp.tell()
887 887 blocksize = self.processingHeaderObj.blockSize
888 888
889 889 #salta el primer bloque de datos
890 890 if csize > self.processingHeaderObj.blockSize:
891 891 self.fp.seek(self.fp.tell() + blocksize)
892 892 else:
893 893 return
894 894
895 895 csize = self.fileSize - self.fp.tell()
896 896 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
897 897 while True:
898 898
899 899 if self.fp.tell()<self.fileSize:
900 900 self.fp.seek(self.fp.tell() + neededsize)
901 901 else:
902 902 self.fp.seek(self.fp.tell() - neededsize)
903 903 break
904 904
905 905 # csize = self.fileSize - self.fp.tell()
906 906 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
907 907 # factor = int(csize/neededsize)
908 908 # if factor > 0:
909 909 # self.fp.seek(self.fp.tell() + factor*neededsize)
910 910
911 911 self.flagIsNewFile = 0
912 912 self.__isFirstTimeOnline = 0
913 913
914 914 def __setNewBlock(self):
915 915
916 916 if self.fp == None:
917 917 return 0
918 918
919 919 if self.online:
920 920 self.__jumpToLastBlock()
921 921
922 922 if self.flagIsNewFile:
923 923 return 1
924 924
925 925 self.lastUTTime = self.basicHeaderObj.utc
926 926 currentSize = self.fileSize - self.fp.tell()
927 927 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
928 928
929 929 if (currentSize >= neededSize):
930 930 self.basicHeaderObj.read(self.fp)
931 931 return 1
932 932
933 933 if self.__waitNewBlock():
934 934 return 1
935 935
936 936 if not(self.setNextFile()):
937 937 return 0
938 938
939 939 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
940 940
941 941 self.flagDiscontinuousBlock = 0
942 942
943 943 if deltaTime > self.maxTimeStep:
944 944 self.flagDiscontinuousBlock = 1
945 945
946 946 return 1
947 947
948 948 def readNextBlock(self):
949 949
950 950 if not(self.__setNewBlock()):
951 951 return 0
952 952
953 953 if not(self.readBlock()):
954 954 return 0
955 955
956 956 self.getBasicHeader()
957 957
958 958 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
959 959 self.processingHeaderObj.dataBlocksPerFile,
960 960 self.dataOut.datatime.ctime())
961 961 return 1
962 962
963 963 def __readFirstHeader(self):
964 964
965 965 self.basicHeaderObj.read(self.fp)
966 966 self.systemHeaderObj.read(self.fp)
967 967 self.radarControllerHeaderObj.read(self.fp)
968 968 self.processingHeaderObj.read(self.fp)
969 969
970 970 self.firstHeaderSize = self.basicHeaderObj.size
971 971
972 972 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
973 973 if datatype == 0:
974 974 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
975 975 elif datatype == 1:
976 976 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
977 977 elif datatype == 2:
978 978 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
979 979 elif datatype == 3:
980 980 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
981 981 elif datatype == 4:
982 982 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
983 983 elif datatype == 5:
984 984 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
985 985 else:
986 986 raise ValueError, 'Data type was not defined'
987 987
988 988 self.dtype = datatype_str
989 989 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
990 990 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
991 991 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
992 992 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
993 993 self.getBlockDimension()
994 994
995 995 def __verifyFile(self, filename, msgFlag=True):
996 996 msg = None
997 997 try:
998 998 fp = open(filename, 'rb')
999 999 currentPosition = fp.tell()
1000 1000 except IOError:
1001 1001 traceback.print_exc()
1002 1002 if msgFlag:
1003 1003 print "[Reading] The file %s can't be opened" % (filename)
1004 1004 return False
1005 1005
1006 1006 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1007 1007
1008 1008 if neededSize == 0:
1009 1009 basicHeaderObj = BasicHeader(LOCALTIME)
1010 1010 systemHeaderObj = SystemHeader()
1011 1011 radarControllerHeaderObj = RadarControllerHeader()
1012 1012 processingHeaderObj = ProcessingHeader()
1013 1013
1014 1014 try:
1015 1015 if not( basicHeaderObj.read(fp) ): raise IOError
1016 1016 if not( systemHeaderObj.read(fp) ): raise IOError
1017 1017 if not( radarControllerHeaderObj.read(fp) ): raise IOError
1018 1018 if not( processingHeaderObj.read(fp) ): raise IOError
1019 1019 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1020 1020
1021 1021 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1022 1022
1023 1023 except IOError:
1024 1024 traceback.print_exc()
1025 1025 sys.exit(0)
1026 1026
1027 1027 if msgFlag:
1028 1028 print "[Reading] The file %s is empty or it hasn't enough data" % filename
1029 1029
1030 1030 fp.close()
1031 1031 return False
1032 1032 else:
1033 1033 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1034 1034
1035 1035 fp.close()
1036 1036 fileSize = os.path.getsize(filename)
1037 1037 currentSize = fileSize - currentPosition
1038 1038 if currentSize < neededSize:
1039 1039 if msgFlag and (msg != None):
1040 1040 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
1041 1041 return False
1042 1042
1043 1043 return True
1044 1044
1045 1045 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1046 1046
1047 1047 dateList = []
1048 1048 pathList = []
1049 1049
1050 1050 multi_path = path.split(',')
1051 1051
1052 1052 if not walk:
1053 1053
1054 1054 for single_path in multi_path:
1055 1055
1056 1056 if not os.path.isdir(single_path):
1057 1057 continue
1058 1058
1059 1059 fileList = glob.glob1(single_path, "*"+ext)
1060 1060
1061 1061 for thisFile in fileList:
1062 1062
1063 1063 if not os.path.isfile(os.path.join(single_path, thisFile)):
1064 1064 continue
1065 1065
1066 1066 if not isRadarFile(thisFile):
1067 1067 continue
1068 1068
1069 1069 if not isFileInDateRange(thisFile, startDate, endDate):
1070 1070 continue
1071 1071
1072 1072 thisDate = getDateFromRadarFile(thisFile)
1073 1073
1074 1074 if thisDate in dateList:
1075 1075 continue
1076 1076
1077 1077 dateList.append(thisDate)
1078 1078 pathList.append(single_path)
1079 1079
1080 1080 else:
1081 1081 for single_path in multi_path:
1082 1082
1083 1083 if not os.path.isdir(single_path):
1084 1084 continue
1085 1085
1086 1086 dirList = []
1087 1087
1088 1088 for thisPath in os.listdir(single_path):
1089 1089
1090 1090 if not os.path.isdir(os.path.join(single_path,thisPath)):
1091 1091 continue
1092 1092
1093 1093 if not isRadarFolder(thisPath):
1094 1094 continue
1095 1095
1096 1096 if not isFolderInDateRange(thisPath, startDate, endDate):
1097 1097 continue
1098 1098
1099 1099 dirList.append(thisPath)
1100 1100
1101 1101 if not dirList:
1102 1102 continue
1103 1103
1104 1104 for thisDir in dirList:
1105 1105
1106 1106 datapath = os.path.join(single_path, thisDir, expLabel)
1107 1107 fileList = glob.glob1(datapath, "*"+ext)
1108 1108
1109 1109 if len(fileList) < 1:
1110 1110 continue
1111 1111
1112 1112 thisDate = getDateFromRadarFolder(thisDir)
1113 1113
1114 1114 pathList.append(datapath)
1115 1115 dateList.append(thisDate)
1116 1116
1117 1117 dateList.sort()
1118 1118
1119 1119 if include_path:
1120 1120 return dateList, pathList
1121 1121
1122 1122 return dateList
1123 1123
1124 1124 def setup(self,
1125 1125 path=None,
1126 1126 startDate=None,
1127 1127 endDate=None,
1128 1128 startTime=datetime.time(0,0,0),
1129 1129 endTime=datetime.time(23,59,59),
1130 1130 set=None,
1131 1131 expLabel = "",
1132 1132 ext = None,
1133 1133 online = False,
1134 1134 delay = 60,
1135 1135 walk = True,
1136 1136 getblock = False,
1137 1137 nTxs = 1):
1138 1138
1139 1139 if path == None:
1140 1140 raise ValueError, "[Reading] The path is not valid"
1141 1141
1142 1142 if ext == None:
1143 1143 ext = self.ext
1144 1144
1145 1145 if online:
1146 1146 print "[Reading] Searching files in online mode..."
1147 1147
1148 1148 for nTries in range( self.nTries ):
1149 1149 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1150 1150
1151 1151 if fullpath:
1152 1152 break
1153 1153
1154 1154 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1155 1155 sleep( self.delay )
1156 1156
1157 1157 if not(fullpath):
1158 1158 print "[Reading] There 'isn't any valid file in %s" % path
1159 1159 return None
1160 1160
1161 1161 self.year = year
1162 1162 self.doy = doy
1163 1163 self.set = set - 1
1164 1164 self.path = path
1165 1165 self.foldercounter = foldercounter
1166 1166 last_set = None
1167 1167
1168 1168 else:
1169 1169 print "[Reading] Searching files in offline mode ..."
1170 1170 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1171 1171 startTime=startTime, endTime=endTime,
1172 1172 set=set, expLabel=expLabel, ext=ext,
1173 1173 walk=walk)
1174 1174
1175 1175 if not(pathList):
1176 1176 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1177 1177 # datetime.datetime.combine(startDate,startTime).ctime(),
1178 1178 # datetime.datetime.combine(endDate,endTime).ctime())
1179 1179 #
1180 1180 sys.exit(-1)
1181 1181
1182 1182
1183 1183 self.fileIndex = -1
1184 1184 self.pathList = pathList
1185 1185 self.filenameList = filenameList
1186 1186 file_name = os.path.basename(filenameList[-1])
1187 1187 basename, ext = os.path.splitext(file_name)
1188 1188 last_set = int(basename[-3:])
1189 1189
1190 1190 self.online = online
1191 1191 self.delay = delay
1192 1192 ext = ext.lower()
1193 1193 self.ext = ext
1194 1194 self.getByBlock = getblock
1195 1195 self.nTxs = int(nTxs)
1196 1196
1197 1197 if not(self.setNextFile()):
1198 1198 if (startDate!=None) and (endDate!=None):
1199 1199 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1200 1200 elif startDate != None:
1201 1201 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1202 1202 else:
1203 1203 print "[Reading] No files"
1204 1204
1205 1205 sys.exit(-1)
1206 1206
1207 1207 # self.getBasicHeader()
1208 1208
1209 1209 if last_set != None:
1210 1210 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1211 1211 return
1212 1212
1213 1213 def getBasicHeader(self):
1214 1214
1215 1215 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1216 1216
1217 1217 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1218 1218
1219 1219 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1220 1220
1221 1221 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1222 1222
1223 1223 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1224 1224
1225 1225 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1226 1226
1227 1227 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1228 1228
1229 1229 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1230 1230
1231 1231
1232 1232 def getFirstHeader(self):
1233 1233
1234 1234 raise ValueError, "This method has not been implemented"
1235 1235
1236 1236 def getData(self):
1237 1237
1238 1238 raise ValueError, "This method has not been implemented"
1239 1239
1240 1240 def hasNotDataInBuffer(self):
1241 1241
1242 1242 raise ValueError, "This method has not been implemented"
1243 1243
1244 1244 def readBlock(self):
1245 1245
1246 1246 raise ValueError, "This method has not been implemented"
1247 1247
1248 1248 def isEndProcess(self):
1249 1249
1250 1250 return self.flagNoMoreFiles
1251 1251
1252 1252 def printReadBlocks(self):
1253 1253
1254 1254 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1255 1255
1256 1256 def printTotalBlocks(self):
1257 1257
1258 1258 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1259 1259
1260 1260 def printNumberOfBlock(self):
1261 1261
1262 1262 if self.flagIsNewBlock:
1263 1263 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1264 1264 self.processingHeaderObj.dataBlocksPerFile,
1265 1265 self.dataOut.datatime.ctime())
1266 1266
1267 1267 def printInfo(self):
1268 1268
1269 1269 if self.__printInfo == False:
1270 1270 return
1271 1271
1272 1272 self.basicHeaderObj.printInfo()
1273 1273 self.systemHeaderObj.printInfo()
1274 1274 self.radarControllerHeaderObj.printInfo()
1275 1275 self.processingHeaderObj.printInfo()
1276 1276
1277 1277 self.__printInfo = False
1278 1278
1279 1279
1280 1280 def run(self, **kwargs):
1281 1281
1282 1282 if not(self.isConfig):
1283 1283
1284 1284 # self.dataOut = dataOut
1285 1285 self.setup(**kwargs)
1286 1286 self.isConfig = True
1287 1287
1288 1288 self.getData()
1289 1289
1290 1290 class JRODataWriter(JRODataIO):
1291 1291
1292 1292 """
1293 1293 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1294 1294 de los datos siempre se realiza por bloques.
1295 1295 """
1296 1296
1297 1297 blockIndex = 0
1298 1298
1299 1299 path = None
1300 1300
1301 1301 setFile = None
1302 1302
1303 1303 profilesPerBlock = None
1304 1304
1305 1305 blocksPerFile = None
1306 1306
1307 1307 nWriteBlocks = 0
1308 1308
1309 1309 fileDate = None
1310 1310
1311 1311 def __init__(self, dataOut=None):
1312 1312 raise ValueError, "Not implemented"
1313 1313
1314 1314
1315 1315 def hasAllDataInBuffer(self):
1316 1316 raise ValueError, "Not implemented"
1317 1317
1318 1318
1319 1319 def setBlockDimension(self):
1320 1320 raise ValueError, "Not implemented"
1321 1321
1322 1322
1323 1323 def writeBlock(self):
1324 1324 raise ValueError, "No implemented"
1325 1325
1326 1326
1327 1327 def putData(self):
1328 1328 raise ValueError, "No implemented"
1329 1329
1330 1330
1331 1331 def getProcessFlags(self):
1332 1332
1333 1333 processFlags = 0
1334 1334
1335 1335 dtype_index = get_dtype_index(self.dtype)
1336 1336 procflag_dtype = get_procflag_dtype(dtype_index)
1337 1337
1338 1338 processFlags += procflag_dtype
1339 1339
1340 1340 if self.dataOut.flagDecodeData:
1341 1341 processFlags += PROCFLAG.DECODE_DATA
1342 1342
1343 1343 if self.dataOut.flagDeflipData:
1344 1344 processFlags += PROCFLAG.DEFLIP_DATA
1345 1345
1346 1346 if self.dataOut.code is not None:
1347 1347 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1348 1348
1349 1349 if self.dataOut.nCohInt > 1:
1350 1350 processFlags += PROCFLAG.COHERENT_INTEGRATION
1351 1351
1352 1352 if self.dataOut.type == "Spectra":
1353 1353 if self.dataOut.nIncohInt > 1:
1354 1354 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1355 1355
1356 1356 if self.dataOut.data_dc is not None:
1357 1357 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1358 1358
1359 1359 if self.dataOut.flagShiftFFT:
1360 1360 processFlags += PROCFLAG.SHIFT_FFT_DATA
1361 1361
1362 1362 return processFlags
1363 1363
1364 1364 def setBasicHeader(self):
1365 1365
1366 1366 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1367 1367 self.basicHeaderObj.version = self.versionFile
1368 1368 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1369 1369
1370 1370 utc = numpy.floor(self.dataOut.utctime)
1371 1371 milisecond = (self.dataOut.utctime - utc)* 1000.0
1372 1372
1373 1373 self.basicHeaderObj.utc = utc
1374 1374 self.basicHeaderObj.miliSecond = milisecond
1375 1375 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1376 1376 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1377 1377 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1378 1378
1379 1379 def setFirstHeader(self):
1380 1380 """
1381 1381 Obtiene una copia del First Header
1382 1382
1383 1383 Affected:
1384 1384
1385 1385 self.basicHeaderObj
1386 1386 self.systemHeaderObj
1387 1387 self.radarControllerHeaderObj
1388 1388 self.processingHeaderObj self.
1389 1389
1390 1390 Return:
1391 1391 None
1392 1392 """
1393 1393
1394 1394 raise ValueError, "No implemented"
1395 1395
1396 1396 def __writeFirstHeader(self):
1397 1397 """
1398 1398 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1399 1399
1400 1400 Affected:
1401 1401 __dataType
1402 1402
1403 1403 Return:
1404 1404 None
1405 1405 """
1406 1406
1407 1407 # CALCULAR PARAMETROS
1408 1408
1409 1409 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1410 1410 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1411 1411
1412 1412 self.basicHeaderObj.write(self.fp)
1413 1413 self.systemHeaderObj.write(self.fp)
1414 1414 self.radarControllerHeaderObj.write(self.fp)
1415 1415 self.processingHeaderObj.write(self.fp)
1416 1416
1417 1417 def __setNewBlock(self):
1418 1418 """
1419 1419 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1420 1420
1421 1421 Return:
1422 1422 0 : si no pudo escribir nada
1423 1423 1 : Si escribio el Basic el First Header
1424 1424 """
1425 1425 if self.fp == None:
1426 1426 self.setNextFile()
1427 1427
1428 1428 if self.flagIsNewFile:
1429 1429 return 1
1430 1430
1431 1431 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1432 1432 self.basicHeaderObj.write(self.fp)
1433 1433 return 1
1434 1434
1435 1435 if not( self.setNextFile() ):
1436 1436 return 0
1437 1437
1438 1438 return 1
1439 1439
1440 1440
1441 1441 def writeNextBlock(self):
1442 1442 """
1443 1443 Selecciona el bloque siguiente de datos y los escribe en un file
1444 1444
1445 1445 Return:
1446 1446 0 : Si no hizo pudo escribir el bloque de datos
1447 1447 1 : Si no pudo escribir el bloque de datos
1448 1448 """
1449 1449 if not( self.__setNewBlock() ):
1450 1450 return 0
1451 1451
1452 1452 self.writeBlock()
1453 1453
1454 1454 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1455 1455 self.processingHeaderObj.dataBlocksPerFile)
1456 1456
1457 1457 return 1
1458 1458
1459 1459 def setNextFile(self):
1460 1460 """
1461 1461 Determina el siguiente file que sera escrito
1462 1462
1463 1463 Affected:
1464 1464 self.filename
1465 1465 self.subfolder
1466 1466 self.fp
1467 1467 self.setFile
1468 1468 self.flagIsNewFile
1469 1469
1470 1470 Return:
1471 1471 0 : Si el archivo no puede ser escrito
1472 1472 1 : Si el archivo esta listo para ser escrito
1473 1473 """
1474 1474 ext = self.ext
1475 1475 path = self.path
1476 1476
1477 1477 if self.fp != None:
1478 1478 self.fp.close()
1479 1479
1480 1480 timeTuple = time.localtime( self.dataOut.utctime)
1481 1481 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1482 1482
1483 1483 fullpath = os.path.join( path, subfolder )
1484 1484 setFile = self.setFile
1485 1485
1486 1486 if not( os.path.exists(fullpath) ):
1487 1487 os.mkdir(fullpath)
1488 1488 setFile = -1 #inicializo mi contador de seteo
1489 1489 else:
1490 1490 filesList = os.listdir( fullpath )
1491 1491 if len( filesList ) > 0:
1492 1492 filesList = sorted( filesList, key=str.lower )
1493 1493 filen = filesList[-1]
1494 1494 # el filename debera tener el siguiente formato
1495 1495 # 0 1234 567 89A BCDE (hex)
1496 1496 # x YYYY DDD SSS .ext
1497 1497 if isNumber( filen[8:11] ):
1498 1498 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1499 1499 else:
1500 1500 setFile = -1
1501 1501 else:
1502 1502 setFile = -1 #inicializo mi contador de seteo
1503 1503
1504 1504 setFile += 1
1505 1505
1506 1506 #If this is a new day it resets some values
1507 1507 if self.dataOut.datatime.date() > self.fileDate:
1508 1508 setFile = 0
1509 1509 self.nTotalBlocks = 0
1510 1510
1511 1511 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1512 1512
1513 1513 filename = os.path.join( path, subfolder, filen )
1514 1514
1515 1515 fp = open( filename,'wb' )
1516 1516
1517 1517 self.blockIndex = 0
1518 1518
1519 1519 #guardando atributos
1520 1520 self.filename = filename
1521 1521 self.subfolder = subfolder
1522 1522 self.fp = fp
1523 1523 self.setFile = setFile
1524 1524 self.flagIsNewFile = 1
1525 1525 self.fileDate = self.dataOut.datatime.date()
1526 1526
1527 1527 self.setFirstHeader()
1528 1528
1529 1529 print '[Writing] Opening file: %s'%self.filename
1530 1530
1531 1531 self.__writeFirstHeader()
1532 1532
1533 1533 return 1
1534 1534
1535 1535 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1536 1536 """
1537 1537 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1538 1538
1539 1539 Inputs:
1540 1540 path : directory where data will be saved
1541 1541 profilesPerBlock : number of profiles per block
1542 1542 set : initial file set
1543 1543 datatype : An integer number that defines data type:
1544 1544 0 : int8 (1 byte)
1545 1545 1 : int16 (2 bytes)
1546 1546 2 : int32 (4 bytes)
1547 1547 3 : int64 (8 bytes)
1548 1548 4 : float32 (4 bytes)
1549 1549 5 : double64 (8 bytes)
1550 1550
1551 1551 Return:
1552 1552 0 : Si no realizo un buen seteo
1553 1553 1 : Si realizo un buen seteo
1554 1554 """
1555 1555
1556 1556 if ext == None:
1557 1557 ext = self.ext
1558 1558
1559 1559 self.ext = ext.lower()
1560 1560
1561 1561 self.path = path
1562 1562
1563 1563 if set is None:
1564 1564 self.setFile = -1
1565 1565 else:
1566 1566 self.setFile = set - 1
1567 1567
1568 1568 self.blocksPerFile = blocksPerFile
1569 1569
1570 1570 self.profilesPerBlock = profilesPerBlock
1571 1571
1572 1572 self.dataOut = dataOut
1573 1573 self.fileDate = self.dataOut.datatime.date()
1574 1574 #By default
1575 1575 self.dtype = self.dataOut.dtype
1576 1576
1577 1577 if datatype is not None:
1578 1578 self.dtype = get_numpy_dtype(datatype)
1579 1579
1580 1580 if not(self.setNextFile()):
1581 1581 print "[Writing] There isn't a next file"
1582 1582 return 0
1583 1583
1584 1584 self.setBlockDimension()
1585 1585
1586 1586 return 1
1587 1587
1588 1588 def run(self, dataOut, **kwargs):
1589 1589
1590 1590 if not(self.isConfig):
1591 1591
1592 1592 self.setup(dataOut, **kwargs)
1593 1593 self.isConfig = True
1594 1594
1595 1595 self.putData()
1596 1596
Show file before | Show file after | Hide comments
@@ -1,1065 +1,1069
1 1 import numpy
2 2
3 3 from jroproc_base import ProcessingUnit, Operation
4 4 from schainpy.model.data.jrodata import Voltage
5 5
6 6 class VoltageProc(ProcessingUnit):
7 7
8 8
9 9 def __init__(self):
10 10
11 11 ProcessingUnit.__init__(self)
12 12
13 13 # self.objectDict = {}
14 14 self.dataOut = Voltage()
15 15 self.flip = 1
16 16
17 17 def run(self):
18 18 if self.dataIn.type == 'AMISR':
19 19 self.__updateObjFromAmisrInput()
20 20
21 21 if self.dataIn.type == 'Voltage':
22 22 self.dataOut.copy(self.dataIn)
23 23
24 24 # self.dataOut.copy(self.dataIn)
25 25
26 26 def __updateObjFromAmisrInput(self):
27 27
28 28 self.dataOut.timeZone = self.dataIn.timeZone
29 29 self.dataOut.dstFlag = self.dataIn.dstFlag
30 30 self.dataOut.errorCount = self.dataIn.errorCount
31 31 self.dataOut.useLocalTime = self.dataIn.useLocalTime
32 32
33 33 self.dataOut.flagNoData = self.dataIn.flagNoData
34 34 self.dataOut.data = self.dataIn.data
35 35 self.dataOut.utctime = self.dataIn.utctime
36 36 self.dataOut.channelList = self.dataIn.channelList
37 37 # self.dataOut.timeInterval = self.dataIn.timeInterval
38 38 self.dataOut.heightList = self.dataIn.heightList
39 39 self.dataOut.nProfiles = self.dataIn.nProfiles
40 40
41 41 self.dataOut.nCohInt = self.dataIn.nCohInt
42 42 self.dataOut.ippSeconds = self.dataIn.ippSeconds
43 43 self.dataOut.frequency = self.dataIn.frequency
44 44
45 45 self.dataOut.azimuth = self.dataIn.azimuth
46 46 self.dataOut.zenith = self.dataIn.zenith
47 47
48 48 self.dataOut.beam.codeList = self.dataIn.beam.codeList
49 49 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
50 50 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
51 51 #
52 52 # pass#
53 53 #
54 54 # def init(self):
55 55 #
56 56 #
57 57 # if self.dataIn.type == 'AMISR':
58 58 # self.__updateObjFromAmisrInput()
59 59 #
60 60 # if self.dataIn.type == 'Voltage':
61 61 # self.dataOut.copy(self.dataIn)
62 62 # # No necesita copiar en cada init() los atributos de dataIn
63 63 # # la copia deberia hacerse por cada nuevo bloque de datos
64 64
65 65 def selectChannels(self, channelList):
66 66
67 67 channelIndexList = []
68 68
69 69 for channel in channelList:
70 70 if channel not in self.dataOut.channelList:
71 71 raise ValueError, "Channel %d is not in %s" %(channel, str(self.dataOut.channelList))
72 72
73 73 index = self.dataOut.channelList.index(channel)
74 74 channelIndexList.append(index)
75 75
76 76 self.selectChannelsByIndex(channelIndexList)
77 77
78 78 def selectChannelsByIndex(self, channelIndexList):
79 79 """
80 80 Selecciona un bloque de datos en base a canales segun el channelIndexList
81 81
82 82 Input:
83 83 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
84 84
85 85 Affected:
86 86 self.dataOut.data
87 87 self.dataOut.channelIndexList
88 88 self.dataOut.nChannels
89 89 self.dataOut.m_ProcessingHeader.totalSpectra
90 90 self.dataOut.systemHeaderObj.numChannels
91 91 self.dataOut.m_ProcessingHeader.blockSize
92 92
93 93 Return:
94 94 None
95 95 """
96 96
97 97 for channelIndex in channelIndexList:
98 98 if channelIndex not in self.dataOut.channelIndexList:
99 99 print channelIndexList
100 100 raise ValueError, "The value %d in channelIndexList is not valid" %channelIndex
101 101
102 102 # nChannels = len(channelIndexList)
103 103 if self.dataOut.flagDataAsBlock:
104 104 """
105 105 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
106 106 """
107 107 data = self.dataOut.data[channelIndexList,:,:]
108 108 else:
109 109 data = self.dataOut.data[channelIndexList,:]
110 110
111 111 self.dataOut.data = data
112 112 self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
113 113 # self.dataOut.nChannels = nChannels
114 114
115 115 return 1
116 116
117 117 def selectHeights(self, minHei=None, maxHei=None):
118 118 """
119 119 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
120 120 minHei <= height <= maxHei
121 121
122 122 Input:
123 123 minHei : valor minimo de altura a considerar
124 124 maxHei : valor maximo de altura a considerar
125 125
126 126 Affected:
127 127 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
128 128
129 129 Return:
130 130 1 si el metodo se ejecuto con exito caso contrario devuelve 0
131 131 """
132 132
133 133 if minHei == None:
134 134 minHei = self.dataOut.heightList[0]
135 135
136 136 if maxHei == None:
137 137 maxHei = self.dataOut.heightList[-1]
138 138
139 139 if (minHei < self.dataOut.heightList[0]):
140 140 minHei = self.dataOut.heightList[0]
141 141 # raise ValueError, "height range [%d,%d] is not valid. Data height range is [%d, %d]" % (minHei,
142 142 # maxHei,
143 143 # self.dataOut.heightList[0],
144 144 # self.dataOut.heightList[-1])
145 145
146 146 if (maxHei > self.dataOut.heightList[-1]):
147 147 maxHei = self.dataOut.heightList[-1]
148 148 # raise ValueError, "some value in (%d,%d) is not valid" % (minHei, maxHei)
149 149
150 150 minIndex = 0
151 151 maxIndex = 0
152 152 heights = self.dataOut.heightList
153 153
154 154 inda = numpy.where(heights >= minHei)
155 155 indb = numpy.where(heights <= maxHei)
156 156
157 157 try:
158 158 minIndex = inda[0][0]
159 159 except:
160 160 minIndex = 0
161 161
162 162 try:
163 163 maxIndex = indb[0][-1]
164 164 except:
165 165 maxIndex = len(heights)
166 166
167 167 self.selectHeightsByIndex(minIndex, maxIndex)
168 168
169 169 return 1
170 170
171 171
172 172 def selectHeightsByIndex(self, minIndex, maxIndex):
173 173 """
174 174 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
175 175 minIndex <= index <= maxIndex
176 176
177 177 Input:
178 178 minIndex : valor de indice minimo de altura a considerar
179 179 maxIndex : valor de indice maximo de altura a considerar
180 180
181 181 Affected:
182 182 self.dataOut.data
183 183 self.dataOut.heightList
184 184
185 185 Return:
186 186 1 si el metodo se ejecuto con exito caso contrario devuelve 0
187 187 """
188 188
189 189 if (minIndex < 0) or (minIndex > maxIndex):
190 190 raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
191 191
192 192 if (maxIndex >= self.dataOut.nHeights):
193 193 maxIndex = self.dataOut.nHeights
194 194 # raise ValueError, "some value in (%d,%d) is not valid" % (minIndex, maxIndex)
195 195
196 196 # nHeights = maxIndex - minIndex + 1
197 197
198 198 #voltage
199 199 if self.dataOut.flagDataAsBlock:
200 200 """
201 201 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
202 202 """
203 203 data = self.dataOut.data[:,minIndex:maxIndex,:]
204 204 else:
205 205 data = self.dataOut.data[:,minIndex:maxIndex]
206 206
207 207 # firstHeight = self.dataOut.heightList[minIndex]
208 208
209 209 self.dataOut.data = data
210 210 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
211 211
212 212 if self.dataOut.nHeights <= 1:
213 213 raise ValueError, "selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights)
214 214
215 215 return 1
216 216
217 217
218 218 def filterByHeights(self, window):
219 219
220 220 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
221 221
222 222 if window == None:
223 223 window = (self.dataOut.radarControllerHeaderObj.txA/self.dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
224 224
225 225 newdelta = deltaHeight * window
226 226 r = self.dataOut.nHeights % window
227 227 newheights = (self.dataOut.nHeights-r)/window
228 228
229 229 if newheights <= 1:
230 230 raise ValueError, "filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(self.dataOut.nHeights, window)
231 231
232 232 if self.dataOut.flagDataAsBlock:
233 233 """
234 234 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
235 235 """
236 236 buffer = self.dataOut.data[:, :, 0:self.dataOut.nHeights-r]
237 237 buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nProfiles,self.dataOut.nHeights/window,window)
238 238 buffer = numpy.sum(buffer,3)
239 239
240 240 else:
241 241 buffer = self.dataOut.data[:,0:self.dataOut.nHeights-r]
242 242 buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nHeights/window,window)
243 243 buffer = numpy.sum(buffer,2)
244 244
245 245 self.dataOut.data = buffer
246 246 self.dataOut.heightList = self.dataOut.heightList[0] + numpy.arange( newheights )*newdelta
247 247 self.dataOut.windowOfFilter = window
248 248
249 249 def setH0(self, h0, deltaHeight = None):
250 250
251 251 if not deltaHeight:
252 252 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
253 253
254 254 nHeights = self.dataOut.nHeights
255 255
256 256 newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
257 257
258 258 self.dataOut.heightList = newHeiRange
259 259
260 260 def deFlip(self, channelList = []):
261 261
262 262 data = self.dataOut.data.copy()
263 263
264 264 if self.dataOut.flagDataAsBlock:
265 265 flip = self.flip
266 266 profileList = range(self.dataOut.nProfiles)
267 267
268 268 if not channelList:
269 269 for thisProfile in profileList:
270 270 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
271 271 flip *= -1.0
272 272 else:
273 273 for thisChannel in channelList:
274 274 if thisChannel not in self.dataOut.channelList:
275 275 continue
276 276
277 277 for thisProfile in profileList:
278 278 data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
279 279 flip *= -1.0
280 280
281 281 self.flip = flip
282 282
283 283 else:
284 284 if not channelList:
285 285 data[:,:] = data[:,:]*self.flip
286 286 else:
287 287 for thisChannel in channelList:
288 288 if thisChannel not in self.dataOut.channelList:
289 289 continue
290 290
291 291 data[thisChannel,:] = data[thisChannel,:]*self.flip
292 292
293 293 self.flip *= -1.
294 294
295 295 self.dataOut.data = data
296 296
297 297 def setRadarFrequency(self, frequency=None):
298 298
299 299 if frequency != None:
300 300 self.dataOut.frequency = frequency
301 301
302 302 return 1
303 303
304 304 class CohInt(Operation):
305 305
306 306 isConfig = False
307 307
308 308 __profIndex = 0
309 309 __withOverapping = False
310 310
311 311 __byTime = False
312 312 __initime = None
313 313 __lastdatatime = None
314 314 __integrationtime = None
315 315
316 316 __buffer = None
317 317
318 318 __dataReady = False
319 319
320 320 n = None
321 321
322 322
323 323 def __init__(self):
324 324
325 325 Operation.__init__(self)
326 326
327 327 # self.isConfig = False
328 328
329 329 def setup(self, n=None, timeInterval=None, overlapping=False, byblock=False):
330 330 """
331 331 Set the parameters of the integration class.
332 332
333 333 Inputs:
334 334
335 335 n : Number of coherent integrations
336 336 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
337 337 overlapping :
338 338
339 339 """
340 340
341 341 self.__initime = None
342 342 self.__lastdatatime = 0
343 343 self.__buffer = None
344 344 self.__dataReady = False
345 345 self.byblock = byblock
346 346
347 347 if n == None and timeInterval == None:
348 348 raise ValueError, "n or timeInterval should be specified ..."
349 349
350 350 if n != None:
351 351 self.n = n
352 352 self.__byTime = False
353 353 else:
354 354 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
355 355 self.n = 9999
356 356 self.__byTime = True
357 357
358 358 if overlapping:
359 359 self.__withOverapping = True
360 360 self.__buffer = None
361 361 else:
362 362 self.__withOverapping = False
363 363 self.__buffer = 0
364 364
365 365 self.__profIndex = 0
366 366
367 367 def putData(self, data):
368 368
369 369 """
370 370 Add a profile to the __buffer and increase in one the __profileIndex
371 371
372 372 """
373 373
374 374 if not self.__withOverapping:
375 375 self.__buffer += data.copy()
376 376 self.__profIndex += 1
377 377 return
378 378
379 379 #Overlapping data
380 380 nChannels, nHeis = data.shape
381 381 data = numpy.reshape(data, (1, nChannels, nHeis))
382 382
383 383 #If the buffer is empty then it takes the data value
384 384 if self.__buffer is None:
385 385 self.__buffer = data
386 386 self.__profIndex += 1
387 387 return
388 388
389 389 #If the buffer length is lower than n then stakcing the data value
390 390 if self.__profIndex < self.n:
391 391 self.__buffer = numpy.vstack((self.__buffer, data))
392 392 self.__profIndex += 1
393 393 return
394 394
395 395 #If the buffer length is equal to n then replacing the last buffer value with the data value
396 396 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
397 397 self.__buffer[self.n-1] = data
398 398 self.__profIndex = self.n
399 399 return
400 400
401 401
402 402 def pushData(self):
403 403 """
404 404 Return the sum of the last profiles and the profiles used in the sum.
405 405
406 406 Affected:
407 407
408 408 self.__profileIndex
409 409
410 410 """
411 411
412 412 if not self.__withOverapping:
413 413 data = self.__buffer
414 414 n = self.__profIndex
415 415
416 416 self.__buffer = 0
417 417 self.__profIndex = 0
418 418
419 419 return data, n
420 420
421 421 #Integration with Overlapping
422 422 data = numpy.sum(self.__buffer, axis=0)
423 423 n = self.__profIndex
424 424
425 425 return data, n
426 426
427 427 def byProfiles(self, data):
428 428
429 429 self.__dataReady = False
430 430 avgdata = None
431 431 # n = None
432 432
433 433 self.putData(data)
434 434
435 435 if self.__profIndex == self.n:
436 436
437 437 avgdata, n = self.pushData()
438 438 self.__dataReady = True
439 439
440 440 return avgdata
441 441
442 442 def byTime(self, data, datatime):
443 443
444 444 self.__dataReady = False
445 445 avgdata = None
446 446 n = None
447 447
448 448 self.putData(data)
449 449
450 450 if (datatime - self.__initime) >= self.__integrationtime:
451 451 avgdata, n = self.pushData()
452 452 self.n = n
453 453 self.__dataReady = True
454 454
455 455 return avgdata
456 456
457 457 def integrate(self, data, datatime=None):
458 458
459 459 if self.__initime == None:
460 460 self.__initime = datatime
461 461
462 462 if self.__byTime:
463 463 avgdata = self.byTime(data, datatime)
464 464 else:
465 465 avgdata = self.byProfiles(data)
466 466
467 467
468 468 self.__lastdatatime = datatime
469 469
470 470 if avgdata is None:
471 471 return None, None
472 472
473 473 avgdatatime = self.__initime
474 474
475 475 deltatime = datatime -self.__lastdatatime
476 476
477 477 if not self.__withOverapping:
478 478 self.__initime = datatime
479 479 else:
480 480 self.__initime += deltatime
481 481
482 482 return avgdata, avgdatatime
483 483
484 484 def integrateByBlock(self, dataOut):
485 485
486 486 times = int(dataOut.data.shape[1]/self.n)
487 487 avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
488 488
489 489 id_min = 0
490 490 id_max = self.n
491 491
492 492 for i in range(times):
493 493 junk = dataOut.data[:,id_min:id_max,:]
494 494 avgdata[:,i,:] = junk.sum(axis=1)
495 495 id_min += self.n
496 496 id_max += self.n
497 497
498 498 timeInterval = dataOut.ippSeconds*self.n
499 499 avgdatatime = (times - 1) * timeInterval + dataOut.utctime
500 500 self.__dataReady = True
501 501 return avgdata, avgdatatime
502 502
503 503 def run(self, dataOut, **kwargs):
504 504
505 505 if not self.isConfig:
506 506 self.setup(**kwargs)
507 507 self.isConfig = True
508 508
509 509 if dataOut.flagDataAsBlock:
510 510 """
511 511 Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
512 512 """
513 513 avgdata, avgdatatime = self.integrateByBlock(dataOut)
514 514 else:
515 515 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
516 516
517 517 # dataOut.timeInterval *= n
518 518 dataOut.flagNoData = True
519 519
520 520 if self.__dataReady:
521 521 dataOut.data = avgdata
522 522 dataOut.nCohInt *= self.n
523 523 dataOut.utctime = avgdatatime
524 524 # dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
525 525 dataOut.flagNoData = False
526 526
527 527 class Decoder(Operation):
528 528
529 529 isConfig = False
530 530 __profIndex = 0
531 531
532 532 code = None
533 533
534 534 nCode = None
535 535 nBaud = None
536 536
537 537
538 538 def __init__(self):
539 539
540 540 Operation.__init__(self)
541 541
542 542 self.times = None
543 543 self.osamp = None
544 544 # self.__setValues = False
545 545 self.isConfig = False
546 546
547 547 def setup(self, code, osamp, dataOut):
548 548
549 549 self.__profIndex = 0
550 550
551 551 self.code = code
552 552
553 553 self.nCode = len(code)
554 554 self.nBaud = len(code[0])
555 555
556 556 if (osamp != None) and (osamp >1):
557 557 self.osamp = osamp
558 558 self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
559 559 self.nBaud = self.nBaud*self.osamp
560 560
561 561 self.__nChannels = dataOut.nChannels
562 562 self.__nProfiles = dataOut.nProfiles
563 563 self.__nHeis = dataOut.nHeights
564 564
565 if self.__nHeis < self.nBaud:
566 print 'IOError: Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud)
567 raise IOError, 'Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud)
568
565 569 if dataOut.flagDataAsBlock:
566 570
567 571 self.ndatadec = self.__nHeis #- self.nBaud + 1
568 572
569 573 self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
570 574
571 575 else:
572 576
573 577 __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
574 578
575 579 __codeBuffer[:,0:self.nBaud] = self.code
576 580
577 581 self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
578 582
579 583 self.ndatadec = self.__nHeis #- self.nBaud + 1
580 584
581 585 self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
582 586
583 587 def convolutionInFreq(self, data):
584 588
585 589 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
586 590
587 591 fft_data = numpy.fft.fft(data, axis=1)
588 592
589 593 conv = fft_data*fft_code
590 594
591 595 data = numpy.fft.ifft(conv,axis=1)
592 596
593 597 datadec = data#[:,:]
594 598
595 599 return datadec
596 600
597 601 def convolutionInFreqOpt(self, data):
598 602
599 603 raise NotImplementedError
600 604
601 605 # fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
602 606 #
603 607 # data = cfunctions.decoder(fft_code, data)
604 608 #
605 609 # datadec = data#[:,:]
606 610 #
607 611 # return datadec
608 612
609 613 def convolutionInTime(self, data):
610 614
611 615 code = self.code[self.__profIndex]
612 616
613 617 for i in range(self.__nChannels):
614 618 self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
615 619
616 620 return self.datadecTime
617 621
618 622 def convolutionByBlockInTime(self, data):
619 623
620 624 repetitions = self.__nProfiles / self.nCode
621 625
622 626 junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
623 627 junk = junk.flatten()
624 628 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
625 629
626 630 for i in range(self.__nChannels):
627 631 for j in range(self.__nProfiles):
628 632 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='same')
629 633
630 634 return self.datadecTime
631 635
632 636 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
633 637
634 638 if dataOut.flagDecodeData:
635 639 print "This data is already decoded, recoding again ..."
636 640
637 641 if not self.isConfig:
638 642
639 643 if code is None:
640 644 code = dataOut.code
641 645 else:
642 646 code = numpy.array(code).reshape(nCode,nBaud)
643 647
644 648 self.setup(code, osamp, dataOut)
645 649
646 650 self.isConfig = True
647 651
648 652 if self.code is None:
649 653 print "Fail decoding: Code is not defined."
650 654 return
651 655
652 656 if dataOut.flagDataAsBlock:
653 657 """
654 658 Decoding when data have been read as block,
655 659 """
656 660 datadec = self.convolutionByBlockInTime(dataOut.data)
657 661
658 662 else:
659 663 """
660 664 Decoding when data have been read profile by profile
661 665 """
662 666 if mode == 0:
663 667 datadec = self.convolutionInTime(dataOut.data)
664 668
665 669 if mode == 1:
666 670 datadec = self.convolutionInFreq(dataOut.data)
667 671
668 672 if mode == 2:
669 673 datadec = self.convolutionInFreqOpt(dataOut.data)
670 674
671 675 dataOut.code = self.code
672 676 dataOut.nCode = self.nCode
673 677 dataOut.nBaud = self.nBaud
674 678
675 679 dataOut.data = datadec
676 680
677 681 dataOut.heightList = dataOut.heightList[0:self.ndatadec]
678 682
679 683 dataOut.flagDecodeData = True #asumo q la data esta decodificada
680 684
681 685 if self.__profIndex == self.nCode-1:
682 686 self.__profIndex = 0
683 687 return 1
684 688
685 689 self.__profIndex += 1
686 690
687 691 return 1
688 692 # dataOut.flagDeflipData = True #asumo q la data no esta sin flip
689 693
690 694
691 695 class ProfileConcat(Operation):
692 696
693 697 isConfig = False
694 698 buffer = None
695 699
696 700 def __init__(self):
697 701
698 702 Operation.__init__(self)
699 703 self.profileIndex = 0
700 704
701 705 def reset(self):
702 706 self.buffer = numpy.zeros_like(self.buffer)
703 707 self.start_index = 0
704 708 self.times = 1
705 709
706 710 def setup(self, data, m, n=1):
707 711 self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
708 712 self.nHeights = data.nHeights
709 713 self.start_index = 0
710 714 self.times = 1
711 715
712 716 def concat(self, data):
713 717
714 718 self.buffer[:,self.start_index:self.profiles*self.times] = data.copy()
715 719 self.start_index = self.start_index + self.nHeights
716 720
717 721 def run(self, dataOut, m):
718 722
719 723 dataOut.flagNoData = True
720 724
721 725 if not self.isConfig:
722 726 self.setup(dataOut.data, m, 1)
723 727 self.isConfig = True
724 728
725 729 if dataOut.flagDataAsBlock:
726 730
727 731 raise ValueError, "ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False"
728 732
729 733 else:
730 734 self.concat(dataOut.data)
731 735 self.times += 1
732 736 if self.times > m:
733 737 dataOut.data = self.buffer
734 738 self.reset()
735 739 dataOut.flagNoData = False
736 740 # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
737 741 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
738 742 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
739 743 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
740 744 dataOut.ippSeconds *= m
741 745
742 746 class ProfileSelector(Operation):
743 747
744 748 profileIndex = None
745 749 # Tamanho total de los perfiles
746 750 nProfiles = None
747 751
748 752 def __init__(self):
749 753
750 754 Operation.__init__(self)
751 755 self.profileIndex = 0
752 756
753 757 def incIndex(self):
754 758
755 759 self.profileIndex += 1
756 760
757 761 if self.profileIndex >= self.nProfiles:
758 762 self.profileIndex = 0
759 763
760 764 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
761 765
762 766 if profileIndex < minIndex:
763 767 return False
764 768
765 769 if profileIndex > maxIndex:
766 770 return False
767 771
768 772 return True
769 773
770 774 def isThisProfileInList(self, profileIndex, profileList):
771 775
772 776 if profileIndex not in profileList:
773 777 return False
774 778
775 779 return True
776 780
777 781 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
778 782
779 783 """
780 784 ProfileSelector:
781 785
782 786 Inputs:
783 787 profileList : Index of profiles selected. Example: profileList = (0,1,2,7,8)
784 788
785 789 profileRangeList : Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
786 790
787 791 rangeList : List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
788 792
789 793 """
790 794
791 795 dataOut.flagNoData = True
792 796
793 797 if nProfiles:
794 798 self.nProfiles = dataOut.nProfiles
795 799 else:
796 800 self.nProfiles = nProfiles
797 801
798 802 if dataOut.flagDataAsBlock:
799 803 """
800 804 data dimension = [nChannels, nProfiles, nHeis]
801 805 """
802 806 if profileList != None:
803 807 dataOut.data = dataOut.data[:,profileList,:]
804 808 dataOut.nProfiles = len(profileList)
805 809 dataOut.profileIndex = dataOut.nProfiles - 1
806 810
807 811 if profileRangeList != None:
808 812 minIndex = profileRangeList[0]
809 813 maxIndex = profileRangeList[1]
810 814
811 815 dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
812 816 dataOut.nProfiles = maxIndex - minIndex + 1
813 817 dataOut.profileIndex = dataOut.nProfiles - 1
814 818
815 819 if rangeList != None:
816 820 raise ValueError, "Profile Selector: Not implemented for rangeList yet"
817 821
818 822 dataOut.flagNoData = False
819 823
820 824 return True
821 825
822 826 else:
823 827 """
824 828 data dimension = [nChannels, nHeis]
825 829
826 830 """
827 831 if profileList != None:
828 832
829 833 dataOut.nProfiles = len(profileList)
830 834
831 835 if self.isThisProfileInList(dataOut.profileIndex, profileList):
832 836 dataOut.flagNoData = False
833 837 dataOut.profileIndex = self.profileIndex
834 838
835 839 self.incIndex()
836 840 return True
837 841
838 842
839 843 if profileRangeList != None:
840 844
841 845 minIndex = profileRangeList[0]
842 846 maxIndex = profileRangeList[1]
843 847
844 848 dataOut.nProfiles = maxIndex - minIndex + 1
845 849
846 850 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
847 851 dataOut.flagNoData = False
848 852 dataOut.profileIndex = self.profileIndex
849 853
850 854 self.incIndex()
851 855 return True
852 856
853 857 if rangeList != None:
854 858
855 859 nProfiles = 0
856 860
857 861 for thisRange in rangeList:
858 862 minIndex = thisRange[0]
859 863 maxIndex = thisRange[1]
860 864
861 865 nProfiles += maxIndex - minIndex + 1
862 866
863 867 dataOut.nProfiles = nProfiles
864 868
865 869 for thisRange in rangeList:
866 870
867 871 minIndex = thisRange[0]
868 872 maxIndex = thisRange[1]
869 873
870 874 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
871 875
872 876 # print "profileIndex = ", dataOut.profileIndex
873 877
874 878 dataOut.flagNoData = False
875 879 dataOut.profileIndex = self.profileIndex
876 880
877 881 self.incIndex()
878 882 break
879 883 return True
880 884
881 885
882 886 if beam != None: #beam is only for AMISR data
883 887 if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
884 888 dataOut.flagNoData = False
885 889 dataOut.profileIndex = self.profileIndex
886 890
887 891 self.incIndex()
888 892 return 1
889 893
890 894 raise ValueError, "ProfileSelector needs profileList, profileRangeList or rangeList parameter"
891 895
892 896 return 0
893 897
894 898
895 899
896 900 class Reshaper(Operation):
897 901
898 902 def __init__(self):
899 903
900 904 Operation.__init__(self)
901 905 self.updateNewHeights = True
902 906
903 907 def run(self, dataOut, shape):
904 908
905 909 if not dataOut.flagDataAsBlock:
906 910 raise ValueError, "Reshaper can only be used when voltage have been read as Block, getBlock = True"
907 911
908 912 if len(shape) != 3:
909 913 raise ValueError, "shape len should be equal to 3, (nChannels, nProfiles, nHeis)"
910 914
911 915 shape_tuple = tuple(shape)
912 916 dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
913 917 dataOut.flagNoData = False
914 918
915 919 if self.updateNewHeights:
916 920
917 921 old_nheights = dataOut.nHeights
918 922 new_nheights = dataOut.data.shape[2]
919 923 factor = 1.0*new_nheights / old_nheights
920 924
921 925 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
922 926
923 927 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * factor
924 928
925 929 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
926 930
927 931 dataOut.nProfiles = dataOut.data.shape[1]
928 932
929 933 dataOut.ippSeconds *= factor
930 934 #
931 935 # import collections
932 936 # from scipy.stats import mode
933 937 #
934 938 # class Synchronize(Operation):
935 939 #
936 940 # isConfig = False
937 941 # __profIndex = 0
938 942 #
939 943 # def __init__(self):
940 944 #
941 945 # Operation.__init__(self)
942 946 # # self.isConfig = False
943 947 # self.__powBuffer = None
944 948 # self.__startIndex = 0
945 949 # self.__pulseFound = False
946 950 #
947 951 # def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
948 952 #
949 953 # #Read data
950 954 #
951 955 # powerdB = dataOut.getPower(channel = channel)
952 956 # noisedB = dataOut.getNoise(channel = channel)[0]
953 957 #
954 958 # self.__powBuffer.extend(powerdB.flatten())
955 959 #
956 960 # dataArray = numpy.array(self.__powBuffer)
957 961 #
958 962 # filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
959 963 #
960 964 # maxValue = numpy.nanmax(filteredPower)
961 965 #
962 966 # if maxValue < noisedB + 10:
963 967 # #No se encuentra ningun pulso de transmision
964 968 # return None
965 969 #
966 970 # maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
967 971 #
968 972 # if len(maxValuesIndex) < 2:
969 973 # #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
970 974 # return None
971 975 #
972 976 # phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
973 977 #
974 978 # #Seleccionar solo valores con un espaciamiento de nSamples
975 979 # pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
976 980 #
977 981 # if len(pulseIndex) < 2:
978 982 # #Solo se encontro un pulso de transmision con ancho mayor a 1
979 983 # return None
980 984 #
981 985 # spacing = pulseIndex[1:] - pulseIndex[:-1]
982 986 #
983 987 # #remover senales que se distancien menos de 10 unidades o muestras
984 988 # #(No deberian existir IPP menor a 10 unidades)
985 989 #
986 990 # realIndex = numpy.where(spacing > 10 )[0]
987 991 #
988 992 # if len(realIndex) < 2:
989 993 # #Solo se encontro un pulso de transmision con ancho mayor a 1
990 994 # return None
991 995 #
992 996 # #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
993 997 # realPulseIndex = pulseIndex[realIndex]
994 998 #
995 999 # period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
996 1000 #
997 1001 # print "IPP = %d samples" %period
998 1002 #
999 1003 # self.__newNSamples = dataOut.nHeights #int(period)
1000 1004 # self.__startIndex = int(realPulseIndex[0])
1001 1005 #
1002 1006 # return 1
1003 1007 #
1004 1008 #
1005 1009 # def setup(self, nSamples, nChannels, buffer_size = 4):
1006 1010 #
1007 1011 # self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
1008 1012 # maxlen = buffer_size*nSamples)
1009 1013 #
1010 1014 # bufferList = []
1011 1015 #
1012 1016 # for i in range(nChannels):
1013 1017 # bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) + numpy.NAN,
1014 1018 # maxlen = buffer_size*nSamples)
1015 1019 #
1016 1020 # bufferList.append(bufferByChannel)
1017 1021 #
1018 1022 # self.__nSamples = nSamples
1019 1023 # self.__nChannels = nChannels
1020 1024 # self.__bufferList = bufferList
1021 1025 #
1022 1026 # def run(self, dataOut, channel = 0):
1023 1027 #
1024 1028 # if not self.isConfig:
1025 1029 # nSamples = dataOut.nHeights
1026 1030 # nChannels = dataOut.nChannels
1027 1031 # self.setup(nSamples, nChannels)
1028 1032 # self.isConfig = True
1029 1033 #
1030 1034 # #Append new data to internal buffer
1031 1035 # for thisChannel in range(self.__nChannels):
1032 1036 # bufferByChannel = self.__bufferList[thisChannel]
1033 1037 # bufferByChannel.extend(dataOut.data[thisChannel])
1034 1038 #
1035 1039 # if self.__pulseFound:
1036 1040 # self.__startIndex -= self.__nSamples
1037 1041 #
1038 1042 # #Finding Tx Pulse
1039 1043 # if not self.__pulseFound:
1040 1044 # indexFound = self.__findTxPulse(dataOut, channel)
1041 1045 #
1042 1046 # if indexFound == None:
1043 1047 # dataOut.flagNoData = True
1044 1048 # return
1045 1049 #
1046 1050 # self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
1047 1051 # self.__pulseFound = True
1048 1052 # self.__startIndex = indexFound
1049 1053 #
1050 1054 # #If pulse was found ...
1051 1055 # for thisChannel in range(self.__nChannels):
1052 1056 # bufferByChannel = self.__bufferList[thisChannel]
1053 1057 # #print self.__startIndex
1054 1058 # x = numpy.array(bufferByChannel)
1055 1059 # self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
1056 1060 #
1057 1061 # deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1058 1062 # dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
1059 1063 # # dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
1060 1064 #
1061 1065 # dataOut.data = self.__arrayBuffer
1062 1066 #
1063 1067 # self.__startIndex += self.__newNSamples
1064 1068 #
1065 1069 # return No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now