##// END OF EJS Templates
schain-jc
RSS
DataIO.py...
Victor Sarmiento -
r50:def509924d6b
parent child
Show More
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (1714 lines changed) Show them Hide them
@@ -8,26 +8,63 import os, sys
8 8 import glob
9 9 import time
10 10 import numpy
11 import fnmatch
12 import time, datetime
11 13
12 14 path = os.path.split(os.getcwd())[0]
13 15 sys.path.append(path)
14 16
15 17 from Model.JROHeader import *
18 from Model.JROData import JROData
16 19
17 def checkForRealPath( path, year, doy, set, ext ):
20
21 def isFileOK(filename):
22 """
23 Determina si la cabecera de un archivo es valido o no, si lo es entonces seria un archivo que podria contener data,
24 si no seria un archivo invalido
25
26 Return:
27 True : si es un archivo valido
28 False : si no es un archivo valido
29
30 Exceptions:
31 Si al leer la cabecera esta no coincide con el tipo de las variables que la contienen entonces se dispara
32 una exception
33 """
34 m_BasicHeader = BasicHeader()
35 m_ProcessingHeader = ProcessingHeader()
36 m_RadarControllerHeader = RadarControllerHeader()
37 m_SystemHeader = SystemHeader()
38 fp = None
39
40 try:
41 fp = open( filename,'rb' )
42 m_BasicHeader.read(fp)
43 m_SystemHeader.read(fp)
44 m_RadarControllerHeader.read(fp)
45 m_ProcessingHeader.read(fp)
46 fp.close()
47 except:
48 if fp != None: fp.close()
49 return False
50
51 return True
52
53
54 def checkForRealPath(path, year, doy, set, ext):
18 55 """
19 56 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
20 57 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
21 58 el path exacto de un determinado file.
22 59
23 60 Example :
24 nombre correcto del file es ../RAWDATA/D2009307/P2009307367
61 nombre correcto del file es .../.../D2009307/P2009307367.ext
25 62
26 63 Entonces la funcion prueba con las siguientes combinaciones
27 ../RAWDATA/d2009307/p2009307367
28 ../RAWDATA/d2009307/P2009307367
29 ../RAWDATA/D2009307/p2009307367
30 ../RAWDATA/D2009307/P2009307367
64 .../.../x2009307/y2009307367.ext
65 .../.../x2009307/Y2009307367.ext
66 .../.../X2009307/y2009307367.ext
67 .../.../X2009307/Y2009307367.ext
31 68 siendo para este caso, la ultima combinacion de letras, identica al file buscado
32 69
33 70 Return:
@@ -38,11 +75,20 def checkForRealPath( path, year, doy, set, ext ):
38 75 filepath = None
39 76 find_flag = False
40 77 filename = None
41
42 for dir in "dD": #barrido por las dos combinaciones posibles de "D"
43 for fil in "dD": #barrido por las dos combinaciones posibles de "D"
78
79 if ext.lower() == ".r": #voltage
80 str1 = "dD"
81 str2 = "dD"
82 elif ext.lower() == ".pdata": #spectra
83 str1 = "dD"
84 str2 = "pP"
85 else:
86 return None, filename
87
88 for dir in str1: #barrido por las dos combinaciones posibles de "D"
89 for fil in str2: #barrido por las dos combinaciones posibles de "D"
44 90 doypath = "%s%04d%03d" % ( dir, year, doy ) #formo el nombre del directorio xYYYYDDD (x=d o x=D)
45 filename = "%s%04d%03d%03d%s" % ( fil, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext (p=d o p=D)
91 filename = "%s%04d%03d%03d%s" % ( fil, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
46 92 filepath = os.path.join( path, doypath, filename ) #formo el path completo
47 93 if os.path.exists( filepath ): #verifico que exista
48 94 find_flag = True
@@ -56,7 +102,7 def checkForRealPath( path, year, doy, set, ext ):
56 102 return filepath, filename
57 103
58 104
59 def isNumber( str ):
105 def isNumber(str):
60 106 """
61 107 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
62 108
@@ -78,8 +124,7 def isNumber( str ):
78 124
79 125 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
80 126 """
81 Esta funcion determina si un archivo de datos en formato Jicamarca(.r) se encuentra
82 o no dentro del rango de fecha especificado.
127 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
83 128
84 129 Inputs:
85 130 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
@@ -116,7 +161,7 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
116 161 return 1
117 162
118 163
119 def getlastFileFromPath( pathList, ext ):
164 def getlastFileFromPath(pathList, ext):
120 165 """
121 166 Depura el pathList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
122 167 al final de la depuracion devuelve el ultimo file de la lista que quedo.
@@ -158,17 +203,19 class DataReader():
158 203 def __init__(self):
159 204 pass
160 205
206
161 207 class DataWriter():
162 208
163 209 def __init__(self):
164 210 pass
165 211
166 class JRODataReader():
212
213 class JRODataReader(DataReader):
167 214
168 215 """
169 Esta clase es usada como la clase padre de las clases VoltageReader and SpectraReader,
216 Esta clase es usada como la clase padre de las clases DataReader,
170 217 contiene todos lo metodos necesarios para leer datos desde archivos en formato
171 Jicamarca (.r o .pdata). La lectura de los datos siempre se realiza por bloques. Los datos
218 jicamarca o pdata (.r o .pdata). La lectura de los datos siempre se realiza por bloques. Los datos
172 219 leidos son array de 3 dimensiones:
173 220 perfiles*alturas*canales
174 221
@@ -178,10 +225,9 class JRODataReader():
178 225 RadarControllerHeader y DataObj. Los tres primeros se usan para almacenar informacion de la
179 226 cabecera de datos (metadata), y el cuarto (DataObj) para obtener y almacenar los datos desde
180 227 el "datablock" cada vez que se ejecute el metodo "getData".
181
182
183 228 """
184 229
230 """
185 231 m_BasicHeader = BasicHeader()
186 232
187 233 m_SystemHeader = SystemHeader()
@@ -243,45 +289,173 class JRODataReader():
243 289
244 290 __nFiles = 3 #number of files for searching
245 291
292 __pts2read = 0
293 __blocksize = 0
294 __utc = 0
295 nBlocks = 0
296 """
246 297 #speed of light
247 298 __c = 3E8
248 299
249 def __init__(self):
250
300 def __init__(self, m_DataObj=None):
251 301 """
252 Inicializador
302 Inicializador de la clase JRODataReader para la lectura de datos.
303
304 Input:
305 m_DataObj : Objeto de la clase JROData. Este objeto sera utilizado para
306 almacenar un perfil de datos cada vez que se haga un requerimiento
307 (getData). El perfil sera obtenido a partir del buffer de datos,
308 si el buffer esta vacio se hara un nuevo proceso de lectura de un
309 bloque de datos.
310 Si este parametro no es pasado se creara uno internamente.
311
312 Variables afectadas:
313 self.m_DataObj
314 self.m_BasicHeader
315 self.m_SystemHeader
316 self.m_RadarControllerHeader
317 self.m_ProcessingHeader
318
319 Return:
320 None
253 321 """
322 if m_DataObj == None:
323 m_DataObj = JROData()
324
325 if not(isinstance(m_DataObj, JROData)):
326 raise ValueError, "in JRODataReader, m_DataObj must be an class object"
327
328 self.m_DataObj = m_DataObj
329
330 self.m_BasicHeader = BasicHeader()
254 331
255 raise ValueError, "This class has not been implemented"
332 self.m_SystemHeader = SystemHeader()
333
334 self.m_RadarControllerHeader = RadarControllerHeader()
335
336 self.m_ProcessingHeader = ProcessingHeader()
256 337
257 def __rdSystemHeader(self,fp=None):
338 self.__fp = None
339
340 self.__fileIndex = None
341
342 self.__startDateTime = None
343
344 self.__endDateTime = None
345
346 self.__dataType = None
347
348 self.__fileSizeByHeader = 0
349
350 self.__pathList = []
351
352 self.filenameList = []
353
354 self.__lastUTTime = 0
355
356 self.__maxTimeStep = 30
357
358 self.__flagIsNewFile = 0
359
360 self.__ippSeconds = 0
361
362 self.flagResetProcessing = 0
363
364 self.flagIsNewBlock = 0
365
366 self.noMoreFiles = 0
367
368 self.nReadBlocks = 0
369
370 self.online = 0
371
372 self.filename = None
373
374 self.fileSize = None
375
376 self.firstHeaderSize = 0
377
378 self.basicHeaderSize = 24
379
380 self.idProfile = 0
381
382 self.datablock = None
383
384 self.__datablockIndex = 9999
385
386 self.__delay = 7 #seconds
387 self.__nTries = 3 #quantity tries
388 self.__nFiles = 3 #number of files for searching
389 self.__year = 0
390 self.__doy = 0
391 self.__set = 0
392 self.__ext = None
393 self.__path = None
394 self.__blocksize = 0
395 self.__utc = 0
396 self.nBlocks = 0
397
398 self.__pts2read = 0
399 self.__pts2read_SelfSpectra = 0
400 self.__pts2read_CrossSpectra = 0
401 self.__pts2read_DCchannels = 0
402 self.__blocksize = 0
403
404 self.__format = None
405
406
407 def __rdSystemHeader(self, fp=None):
258 408
259 409 if fp == None:
260 410 fp = self.__fp
261 411
262 412 self.m_SystemHeader.read(fp)
413
263 414
264 def __rdRadarControllerHeader(self,fp=None):
415 def __rdRadarControllerHeader(self, fp=None):
265 416 if fp == None:
266 417 fp = self.__fp
267 418
268 419 self.m_RadarControllerHeader.read(fp)
420
269 421
270 def __rdProcessingHeader(self,fp=None):
422 def __rdProcessingHeader(self, fp=None):
271 423 if fp == None:
272 424 fp = self.__fp
273 425
274 426 self.m_ProcessingHeader.read(fp)
275 427
428
276 429 def __rdBasicHeader(self, fp=None):
277 430
278 431 if fp == None:
279 432 fp = self.__fp
280 433
281 434 self.m_BasicHeader.read(fp)
435
282 436
283 437 def __readFirstHeader(self):
284
438 """
439 Lectura del First Header, es decir el Basic Header y el Long Header
440
441 Affected:
442 self.m_BasicHeader
443 self.m_SystemHeader
444 self.m_RadarControllerHeader
445 self.m_ProcessingHeader
446 self.firstHeaderSize
447 self.__heights
448 self.__dataType
449 self.__fileSizeByHeader
450 self.__ippSeconds
451 self.nChannels
452 self.nPairs
453 self.__pts2read_SelfSpectra
454 self.__pts2read_CrossSpectra
455
456 Return:
457 None
458 """
285 459 self.__rdBasicHeader()
286 460 self.__rdSystemHeader()
287 461 self.__rdRadarControllerHeader()
@@ -317,13 +491,35 class JRODataReader():
317 491 self.__heights = numpy.arange(xi, xf, step)
318 492 self.__dataType = tmp
319 493 self.__fileSizeByHeader = self.m_ProcessingHeader.dataBlocksPerFile * self.m_ProcessingHeader.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.m_ProcessingHeader.dataBlocksPerFile - 1)
320 self.__ippSeconds = 2*1000*self.m_RadarControllerHeader.ipp/self.__c
494 self.__ippSeconds = 2 * 1000 * self.m_RadarControllerHeader.ipp / self.__c
321 495
322 self.__pts2read = self.m_ProcessingHeader.profilesPerBlock * self.m_ProcessingHeader.numHeights * self.m_SystemHeader.numChannels
323 self.__blocksize = self.__pts2read
496 if self.__format == "jicamarca":
497 self.__pts2read = self.m_ProcessingHeader.profilesPerBlock * self.m_ProcessingHeader.numHeights * self.m_SystemHeader.numChannels
498 self.__blocksize = self.__pts2read
324 499
500 elif self.__format == "pdata":
501 self.nChannels = 0
502 self.nPairs = 0
503
504 for i in range( 0, self.m_ProcessingHeader.totalSpectra*2, 2 ):
505 if self.m_ProcessingHeader.spectraComb[i] == self.m_ProcessingHeader.spectraComb[i+1]:
506 self.nChannels = self.nChannels + 1
507 else:
508 self.nPairs = self.nPairs + 1
509
510 pts2read = self.m_ProcessingHeader.profilesPerBlock * self.m_ProcessingHeader.numHeights
511 self.__pts2read_SelfSpectra = int( pts2read * self.nChannels )
512 self.__pts2read_CrossSpectra = int( pts2read * self.nPairs )
513 self.__pts2read_DCchannels = int( self.m_ProcessingHeader.numHeights * self.m_SystemHeader.numChannels )
514
515 self.__blocksize = self.__pts2read_SelfSpectra + self.__pts2read_CrossSpectra + self.__pts2read_DCchannels
516
517 self.m_DataObj.nChannels = self.nChannels
518 self.m_DataObj.nPairs = self.nPairs
519
325 520
326 def __setNextFileOnline( self ):
521
522 def __setNextFileOnline(self):
327 523 """
328 524 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
329 525 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
@@ -347,17 +543,13 class JRODataReader():
347 543 countFiles = 0
348 544 countTries = 0
349 545
350 fileStatus = 0
351 546 notFirstTime_flag = False
352 bChangeDir = False
353
354 fileSize = 0
355 fp = None
547 fileOk_flag = False
548 changeDir_flag = False
549 self.__flagIsNewFile = 0
356 550
357 self.__flagNewFile = 0
358
359 #este loop permite llevar la cuenta de intentos, de files y carpetas, si no encuentra alguno sale del bucle
360 while( True ):
551 while( True ): #este loop permite llevar la cuenta de intentos, de files y carpetas,
552 #si no encuentra alguno sale del bucle
361 553 countFiles += 1
362 554
363 555 if countFiles > (self.__nFiles + 1):
@@ -368,94 +560,103 class JRODataReader():
368 560 if countFiles > self.__nFiles: #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
369 561 self.__set = 0
370 562 self.__doy += 1
371 bChangeDir = True
563 changeDir_flag = True
372 564
373 565 file = None
374 566 filename = None
567 fileOk_flag = False
375 568
376 countTries = 0
377
378 #espero hasta encontrar el 1er file disponible
379 while( True ):
380
381 countTries += 1
382 if( countTries >= self.__nTries ): #checkeo que no haya ido mas alla de la cantidad de intentos
383 break
384
385 file, filename = checkForRealPath( self.__path, self.__year, self.__doy, self.__set, self.__ext )
386 if file != None:
387 break
388
389 if notFirstTime_flag: #este flag me sirve solo para esperar por el 1er file, en lo siguientes no espera solo checkea si existe o no
390 countTries = self.__nTries
391 print "\tsearching next \"%s\" file ..." % filename
392 break
393
394 print "\twaiting new \"%s\" file ..." % filename
395 time.sleep( self.__delay )
396
397 if countTries >= self.__nTries: #se realizaron n intentos y no hubo un file nuevo
398 notFirstTime_flag = True
399 continue #vuelvo al inico del while principal
569 #busca el 1er file disponible
570 file, filename = checkForRealPath( self.__path, self.__year, self.__doy, self.__set, self.__ext )
571
572 if file == None:
573 if notFirstTime_flag: #si no es la primera vez que busca el file entonces no espera y busca for el siguiente file
574 print "\tsearching next \"%s\" file ..." % ( filename )
575 continue
576 else: #si es la primera vez que busca el file entonces espera self.__nTries veces hasta encontrarlo o no
577 for nTries in range( self.__nTries ):
578 print "\twaiting new \"%s\" file, try %03d ..." % ( filename, nTries+1 )
579 time.sleep( self.__delay )
580
581 file, filename = checkForRealPath( self.__path, self.__year, self.__doy, self.__set, self.__ext )
582 if file != None:
583 fileOk_flag = True
584 break
585
586 if not( fileOk_flag ): #no encontro ningun file valido a leer despues de haber esperado alguno
587 notFirstTime_flag = True
588 continue
589
590 #una vez que se obtuvo el 1er file valido se procede a checkear si si tamanho es suficiente para empezar a leerlo
591 currentSize = os.path.getsize( file )
592 neededSize = self.m_ProcessingHeader.blockSize + self.firstHeaderSize
400 593
401 countTries = 0
594 #si el tamanho es suficiente entonces deja de buscar
595 if currentSize > neededSize:
596 fileOk_flag = True
597 break
402 598
403 #una vez que se obtuvo el 1er file valido se procede a checkear su contenido, y se espera una cierta cantidad
404 #de tiempo por una cierta cantidad de veces hasta que el contenido del file sea un contenido valido
405 while( True ):
406 countTries += 1
407 if countTries > self.__nTries:
408 break
409
410 try:
411 fp = open(file)
412 except:
413 print "The file \"%s\" can't be opened" % file
414 break
415
416 fileSize = os.path.getsize( file )
417 currentSize = fileSize - fp.tell()
599 fileOk_flag = False
600 #si el file no tiene el tamanho necesario se espera una cierta cantidad de tiempo
601 #por una cierta cantidad de veces hasta que el contenido del file sea valido
602 if changeDir_flag: #si al buscar un file cambie de directorio ya no espero y sigo con el siguiente file
603 print "\tsearching next \"%s\" file ..." % filename
604 changeDir_flag = False
605 continue
606
607 for nTries in range( self.__nTries ):
608 print "\twaiting for the First Header block of \"%s\" file, try %03d ..." % ( filename, nTries+1 )
609 time.sleep( self.__delay )
610
611 currentSize = os.path.getsize( file )
418 612 neededSize = self.m_ProcessingHeader.blockSize + self.firstHeaderSize
419 613
420 614 if currentSize > neededSize:
421 fileStatus = 1
615 fileOk_flag = True
422 616 break
423 617
424 fp.close()
425
426 if bChangeDir: #si al buscar un file cambie de directorio ya no espero y salgo del bucle while
427 print "\tsearching next \"%s\" file ..." % filename
428 break
429
430 print "\twaiting for block of \"%s\" file ..." % filename
431 time.sleep( self.__delay )
432
433 if fileStatus == 1:
618 if fileOk_flag: #si encontro un file valido sale del bucle y deja de buscar
434 619 break
435 620
436 621 print "Skipping the file \"%s\" due to this files is empty" % filename
437 622 countFiles = 0
438 623
439
440 if fileStatus == 1:
441 self.fileSize = fileSize
624 if fileOk_flag:
625 self.fileSize = os.path.getsize( file )
442 626 self.filename = file
443 self.__flagNewFile = 1
444 self.__fp = fp
445 self.flagNoMoreFiles = 0
627 self.__flagIsNewFile = 1
628 if self.__fp != None: self.__fp.close()
629 self.__fp = open(file)
630 self.noMoreFiles = 0
446 631 print 'Setting the file: %s' % file
447 632 else:
448 633 self.fileSize = 0
449 634 self.filename = None
450 635 self.__fp = None
451 self.flagNoMoreFiles = 1
636 self.noMoreFiles = 1
452 637 print 'No more Files'
453 638
454 return fileStatus
639 return fileOk_flag
455 640
456 641
457 642 def __setNextFileOffline(self):
643 """
644 Busca el siguiente file dentro de un folder que tenga suficiente data para ser leida
645
646 Affected:
647 self.__flagIsNewFile
648 self.__fileIndex
649 self.filename
650 self.fileSize
651 self.__fp
652
653 Return:
654 0 : si un determinado file no puede ser abierto
655 1 : si el file fue abierto con exito
458 656
657 Excepciones:
658 Si un determinado file no puede ser abierto
659 """
459 660 idFile = self.__fileIndex
460 661 while(True):
461 662
@@ -463,6 +664,7 class JRODataReader():
463 664
464 665 if not(idFile < len(self.__filenameList)):
465 666 self.flagNoMoreFiles = 1
667 print 'No more Files'
466 668 return 0
467 669
468 670 filename = self.__filenameList[idFile]
@@ -491,6 +693,7 class JRODataReader():
491 693 print 'Setting the file: %s'%self.filename
492 694
493 695 return 1
696
494 697
495 698 def __setNextFile( self ):
496 699 """
@@ -515,15 +718,18 class JRODataReader():
515 718 else:
516 719 newFile = self.__setNextFileOffline()
517 720
721 if self.noMoreFiles:
722 sys.exit(0)
723
518 724 if not(newFile):
519 725 return 0
520 726
521 727 self.__readFirstHeader()
522
728 self.nBlocks = 0
523 729 return 1
524 730
525 731
526 def __setNewBlock( self ):
732 def __setNewBlock(self):
527 733 """
528 734 Lee el Basic Header y posiciona le file pointer en la posicion inicial del bloque a leer
529 735
@@ -539,7 +745,7 class JRODataReader():
539 745 if self.__fp == None:
540 746 return 0
541 747
542 if self.__flagNewFile:
748 if self.__flagIsNewFile:
543 749 return 1
544 750
545 751 currentSize = self.fileSize - self.__fp.tell()
@@ -549,107 +755,199 class JRODataReader():
549 755 if ( currentSize >= neededSize ):
550 756 self.__rdBasicHeader()
551 757 return 1
552 elif self.online:
553 nTries = 0
554 while( nTries < self.__nTries ):
555 nTries += 1
556 print "Waiting for the next block, try %03d ..." % nTries
758
759 #si es OnLine y ademas aun no se han leido un bloque completo entonces se espera por uno valido
760 elif (self.nBlocks != self.m_ProcessingHeader.dataBlocksPerFile) and self.online:
761 for nTries in range( self.__nTries ):
762
763 fpointer = self.__fp.tell()
764 self.__fp.close()
765
766 print "\tWaiting for the next block, try %03d ..." % (nTries+1)
557 767 time.sleep( self.__delay )
558
559 fileSize = os.path.getsize(self.filename)
560 currentSize = fileSize - self.__fp.tell()
768
769 self.__fp = open( self.filename, 'rb' )
770 self.__fp.seek( fpointer )
771
772 self.fileSize = os.path.getsize( self.filename )
773 currentSize = self.fileSize - self.__fp.tell()
561 774 neededSize = self.m_ProcessingHeader.blockSize + self.basicHeaderSize
562 775
563 776 if ( currentSize >= neededSize ):
564 777 self.__rdBasicHeader()
565 778 return 1
566
779
567 780 #Setting new file
568 781 if not( self.__setNextFile() ):
569 782 return 0
570 783
571 784 deltaTime = self.m_BasicHeader.utc - self.__lastUTTime # check this
572 785
573 self.flagNoContinuousBlock = 0
786 self.flagResetProcessing = 0
574 787
575 788 if deltaTime > self.__maxTimeStep:
576 self.flagNoContinuousBlock = 1
577 self.nReadBlocks = 0
789 self.flagResetProcessing = 1
790 #self.nReadBlocks = 0
578 791
579 792 return 1
793
580 794
581 def __readBlock(self):
795 def __readBlockVoltage(self):
582 796 """
583 797 __readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
584 798 (self.__fp) y actualiza todos los parametros relacionados al bloque de datos
585 799 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
586 800 es seteado a 0
587 801
588
589 802 Inputs:
590 803 None
591 804
592 805 Return:
593 806 None
594 807
595 Variables afectadas:
596
808 Affected:
597 809 self.__datablockIndex
598
599 810 self.datablock
600
601 self.__flagNewFile
602
603 self.__flagNewBlock
604
811 self.__flagIsNewFile
812 self.idProfile
813 self.flagIsNewBlock
605 814 self.nReadBlocks
606 815
816 Exceptions:
817 Si un bloque leido no es un bloque valido
607 818 """
608
609 #pts2read = self.m_ProcessingHeader.profilesPerBlock*self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels
610
819 blockOk_flag = False
611 820 fpointer = self.__fp.tell()
612 821
613 822 junk = numpy.fromfile( self.__fp, self.__dataType, self.__pts2read )
614 823
615 824 if self.online:
616 825 if junk.size != self.__blocksize:
617 nTries = 0
618 while( nTries < self.__nTries ):
619 nTries += 1
620 print "Waiting for the next block, try %03d ..." % nTries
826 for nTries in range( self.__nTries ):
827 print "\tWaiting for the next block, try %03d ..." % (nTries+1)
621 828 time.sleep( self.__delay )
622 829 self.__fp.seek( fpointer )
623 830 fpointer = self.__fp.tell()
624 831 junk = numpy.fromfile( self.__fp, self.__dataType, self.__pts2read )
625 832 if junk.size == self.__blocksize:
626 nTries = 0
833 blockOk_flag = True
627 834 break
628 if nTries > 0:
629 return
835
836 if not( blockOk_flag ):
837 return 0
630 838
631 junk = junk.reshape( (self.m_ProcessingHeader.profilesPerBlock, self.m_ProcessingHeader.numHeights, self.m_SystemHeader.numChannels) )
839 try:
840 junk = junk.reshape( (self.m_ProcessingHeader.profilesPerBlock, self.m_ProcessingHeader.numHeights, self.m_SystemHeader.numChannels) )
841 except:
842 print "Data file %s is invalid" % self.filename
843 return 0
632 844
633 data = junk['real'] + junk['imag']*1j
845 self.datablock = junk['real'] + junk['imag']*1j
634 846
635 847 self.__datablockIndex = 0
848 self.__flagIsNewFile = 0
849 self.idProfile = 0
850 self.flagIsNewBlock = 1
851
852 self.nReadBlocks += 1
853 self.nBlocks += 1
854
855 return 1
856
857
858 def __readBlockSpectra(self):
859 """
860 Lee el bloque de datos desde la posicion actual del puntero del archivo
861 (self.__fp) y actualiza todos los parametros relacionados al bloque de datos
862 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
863 es seteado a 0
636 864
637 self.datablock = data
865 Return: None
638 866
639 self.__flagNewFile = 0
867 Variables afectadas:
868 self.__datablockIndex
869 self.__flagIsNewFile
870 self.flagIsNewBlock
871 self.nReadBlocks
872 self.__data_spc
873 self.__data_cspc
874 self.__data_dc
875
876 Exceptions:
877 Si un bloque leido no es un bloque valido
878 """
879 blockOk_flag = False
880 fpointer = self.__fp.tell()
881
882 spc = numpy.fromfile( self.__fp, self.__dataType[0], self.__pts2read_SelfSpectra )
883 cspc = numpy.fromfile( self.__fp, self.__dataType, self.__pts2read_CrossSpectra )
884 dc = numpy.fromfile( self.__fp, self.__dataType, self.__pts2read_DCchannels ) #int(self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels) )
885
886 if self.online:
887 if (spc.size + cspc.size + dc.size) != self.__blocksize:
888 for nTries in range( self.__nTries ):
889 #nTries = 0
890 #while( nTries < self.__nTries ):
891 #nTries += 1
892 print "\tWaiting for the next block, try %03d ..." % (nTries+1)
893 time.sleep( self.__delay )
894 self.__fp.seek( fpointer )
895 fpointer = self.__fp.tell()
896 spc = numpy.fromfile( self.__fp, self.__dataType[0], self.__pts2read_SelfSpectra )
897 cspc = numpy.fromfile( self.__fp, self.__dataType, self.__pts2read_CrossSpectra )
898 dc = numpy.fromfile( self.__fp, self.__dataType, self.__pts2read_DCchannels ) #int(self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels) )
899
900 if (spc.size + cspc.size + dc.size) == self.__blocksize:
901 blockOk_flag = True
902 break
903 #if (spc.size + cspc.size + dc.size) == self.__blocksize:
904 # nTries = 0
905 # break
906 if not( blockOk_flag ):
907 return 0
908 #if nTries > 0:
909 # return 0
910
911 try:
912 spc = spc.reshape( (self.nChannels, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D
913 cspc = cspc.reshape( (self.nPairs, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D
914 dc = dc.reshape( (self.m_SystemHeader.numChannels, self.m_ProcessingHeader.numHeights) ) #transforma a un arreglo 2D
915 except:
916 print "Data file %s is invalid" % self.filename
917 return 0
640 918
641 self.__flagNewBlock = 1
919 if not( self.m_ProcessingHeader.shif_fft ):
920 spc = numpy.roll( spc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
921 cspc = numpy.roll( cspc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
642 922
923 spc = numpy.transpose( spc, (0,2,1) )
924 cspc = numpy.transpose( cspc, (0,2,1) )
925 #dc = numpy.transpose(dc, (0,2,1))
926
927 self.__data_spc = spc
928 self.__data_cspc = cspc['real'] + cspc['imag']*1j
929 self.__data_dc = dc['real'] + dc['imag']*1j
930
931 self.datablock_id = 0
932 self.__flagIsNewFile = 0
933 self.flagIsNewBlock = 1
934
643 935 self.nReadBlocks += 1
936 self.nBlocks += 1
937
938 return 1
939
644 940
645 941 def __hasNotDataInBuffer(self):
646 if self.__datablockIndex >= self.m_ProcessingHeader.profilesPerBlock:
647 return 1
648
649 return 0
942 if self.__format == "jicamarca":
943 if self.__datablockIndex >= self.m_ProcessingHeader.profilesPerBlock:
944 return 1
945 else: return 0
946
947 return 1
650 948
651 949
652 def __searchFilesOnLine( self, path, startDateTime=None, ext = ".r" ):
950 def __searchFilesOnLine( self, path, startDateTime=None, ext = ".pdata" ):
653 951 """
654 952 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
655 953 devuelve el archivo encontrado ademas de otros datos.
@@ -679,7 +977,7 class JRODataReader():
679 977
680 978 dirList = sorted( dirList, key=str.lower ) #para que quede ordenado al margen de si el nombre esta en mayusculas o minusculas, utilizo la funcion sorted
681 979 if len(dirList) > 0 :
682 directory = dirList[-1]
980 directory = dirList[-1] #me quedo con el ultimo directorio de una carpeta
683 981 else:
684 982 year = startDateTime.timetuple().tm_year
685 983 doy = startDateTime.timetuple().tm_yday
@@ -814,17 +1112,11 class JRODataReader():
814 1112
815 1113 file = os.path.join( path, dirfilename, filename )
816 1114
817 nTries = 0
818 while(True):
819
820 nTries += 1
821 if nTries > self.__nTries:
822 break
823
1115 for nTries in range( self.__nTries+1 ):
824 1116 try:
825 1117 fp = open( file,'rb' ) #lectura binaria
826 1118 except:
827 raise IOError, "The file %s can't be opened" %(file)
1119 raise IOError, "The file %s can't be opened" % (file)
828 1120
829 1121 try:
830 1122 m_BasicHeader.read(fp)
@@ -836,16 +1128,19 class JRODataReader():
836 1128 if m_BasicHeader.size > 24:
837 1129 break
838 1130
839 print 'waiting for new block: try %02d' % ( nTries )
840 time.sleep( self.__delay)
841
1131 if nTries >= self.__nTries: #si ya espero la cantidad de veces necesarias entonces ya no vuelve a esperar
1132 break
1133
1134 print '\twaiting for new block of file %s: try %02d' % ( file, nTries )
1135 time.sleep( self.__delay )
1136
842 1137 if m_BasicHeader.size <= 24:
843 1138 return 0
844 1139
845 1140 return 1
846 1141
847 1142
848 def setup(self, path, startDateTime, endDateTime=None, set=None, expLabel = "", ext = ".r", online = 0):
1143 def setup(self, path, startDateTime, endDateTime=None, set=0, expLabel = "", ext = None, online = 0):
849 1144 """
850 1145 setup configura los parametros de lectura de la clase VoltageReader.
851 1146
@@ -889,24 +1184,30 class JRODataReader():
889 1184 self.online
890 1185 """
891 1186 if online:
892 nTries = 0
893 while( nTries < self.__nTries ):
894 nTries += 1
895 subfolder = "D%04d%03d" % ( startDateTime.timetuple().tm_year, startDateTime.timetuple().tm_yday )
896 year, doy, set, filename, dirfilename = self.__searchFilesOnLine( path, startDateTime, ext )
897 if filename == None:
898 file = os.path.join( path, subfolder )
899 print "Searching first file in \"%s\", try %03d ..." % ( file, nTries )
900 time.sleep( self.__delay )
901 else:
902 break
903
904 if filename == None:
905 print "No files On Line"
1187 fileOK_flag = False
1188 subfolder = "D%04d%03d" % ( startDateTime.timetuple().tm_year, startDateTime.timetuple().tm_yday )
1189 file = os.path.join( path, subfolder )
1190
1191 for nTries in range( self.__nTries+1 ): #espera por el 1er file
1192 year, doy, set, filename, dirfilename = self.__searchFilesOnLine( path, startDateTime, ext )
1193
1194 if filename != None:
1195 if isFileOK( os.path.join( path,dirfilename,filename ) ):
1196 fileOK_flag = True
1197 break
1198
1199 if nTries >= self.__nTries: #si ya espero la cantidad de veces necesarias entonces ya no vuelve a esperar
1200 break
1201
1202 print "Searching first file in \"%s\", try %03d ..." % ( file, nTries+1 )
1203 time.sleep( self.__delay )
1204
1205 if not( fileOK_flag ): #filename == None:
1206 print "No files on line or invalid first file"
906 1207 return 0
907 1208
908 1209 if self.__initFilesOnline( path, dirfilename, filename ) == 0:
909 print "The file %s hasn't enough data"
1210 print "The file %s hasn't enough data" % filename
910 1211 return 0
911 1212
912 1213 self.__year = year
@@ -916,13 +1217,29 class JRODataReader():
916 1217
917 1218 else:
918 1219 pathList, filenameList = self.__searchFilesOffLine( path, startDateTime, endDateTime, set, expLabel, ext )
919 self.__fileIndex = -1
1220 self.__fileIndex = -1
920 1221 self.__pathList = pathList
921 self.__filenameList = filenameList
1222 self.filenameList = filenameList
922 1223
923 1224 self.online = online
924 1225 self.__ext = ext
925 1226
1227 ext = ext.lower()
1228
1229 if ext == '.hdf5':
1230 print 'call hdf5 library'
1231 return 0
1232
1233 elif ext == '.r':
1234 self.__format = 'jicamarca'
1235
1236 elif ext == '.pdata':
1237 self.__format = 'pdata'
1238
1239 else:
1240 print 'unknow format !!!'
1241 return 0
1242
926 1243 if not( self.__setNextFile() ):
927 1244 if (startDateTime != None) and (endDateTime != None):
928 1245 print "No files in range: %s - %s" %(startDateTime.ctime(), endDateTime.ctime())
@@ -943,11 +1260,11 class JRODataReader():
943 1260 self.endDoy = endDateTime.timetuple().tm_yday
944 1261 #call fillHeaderValues() - to Data Object
945 1262
946 self.m_Voltage.m_BasicHeader = self.m_BasicHeader.copy()
947 self.m_Voltage.m_ProcessingHeader = self.m_ProcessingHeader.copy()
948 self.m_Voltage.m_RadarControllerHeader = self.m_RadarControllerHeader.copy()
949 self.m_Voltage.m_SystemHeader = self.m_SystemHeader.copy()
950 self.m_Voltage.dataType = self.__dataType
1263 self.m_DataObj.m_BasicHeader = self.m_BasicHeader.copy()
1264 self.m_DataObj.m_ProcessingHeader = self.m_ProcessingHeader.copy()
1265 self.m_DataObj.m_RadarControllerHeader = self.m_RadarControllerHeader.copy()
1266 self.m_DataObj.m_SystemHeader = self.m_SystemHeader.copy()
1267 self.m_DataObj.dataType = self.__dataType
951 1268
952 1269 return 1
953 1270
@@ -965,7 +1282,11 class JRODataReader():
965 1282 if not(self.__setNewBlock()):
966 1283 return 0
967 1284
968 self.__readBlock()
1285 if self.__format == "jicamarca":
1286 self.__readBlockVoltage()
1287
1288 elif self.__format == "pdata":
1289 self.__readBlockSpectra()
969 1290
970 1291 self.__lastUTTime = self.m_BasicHeader.utc
971 1292
@@ -994,521 +1315,540 class JRODataReader():
994 1315 self.flagNoContinuousBlock
995 1316 self.__flagNewBlock
996 1317 """
997 self.flagNoContinuousBlock = 0
998 self.__flagNewBlock = 0
1318 if self.noMoreFiles: return 0
1319
1320 self.flagResetProcessing = 0
1321 self.flagIsNewBlock = 0
999 1322
1000 if self.__hasNotDataInBuffer():
1323 if self.__hasNotDataInBuffer():
1001 1324
1002 self.readNextBlock()
1003
1004 self.m_Voltage.m_BasicHeader = self.m_BasicHeader.copy()
1005 self.m_Voltage.m_ProcessingHeader = self.m_ProcessingHeader.copy()
1006 self.m_Voltage.m_RadarControllerHeader = self.m_RadarControllerHeader.copy()
1007 self.m_Voltage.m_SystemHeader = self.m_SystemHeader.copy()
1008 self.m_Voltage.heights = self.__heights
1009 self.m_Voltage.dataType = self.__dataType
1010
1011 if self.flagNoMoreFiles == 1:
1325 if not( self.readNextBlock() ):
1326 self.__setNextFile()
1327 return 0
1328
1329 self.m_DataObj.m_BasicHeader = self.m_BasicHeader.copy()
1330 self.m_DataObj.m_ProcessingHeader = self.m_ProcessingHeader.copy()
1331 self.m_DataObj.m_RadarControllerHeader = self.m_RadarControllerHeader.copy()
1332 self.m_DataObj.m_SystemHeader = self.m_SystemHeader.copy()
1333 self.m_DataObj.heights = self.__heights
1334 self.m_DataObj.dataType = self.__dataType
1335
1336 if self.noMoreFiles == 1:
1012 1337 print 'Process finished'
1013 return None
1338 return 0
1014 1339
1015 1340 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
1016 1341
1017 time = self.m_BasicHeader.utc + self.__datablockIndex * self.__ippSeconds
1018 self.m_Voltage.m_BasicHeader.utc = time
1342 if self.__format == "jicamarca":
1343 if self.datablock == None:
1344 self.m_Voltage.flagNoData = True
1345 return 0
1346
1347 time = self.m_BasicHeader.utc + self.__datablockIndex * self.__ippSeconds
1348 self.__utc = time
1349 #self.m_DataObj.m_BasicHeader.utc = time
1350
1351 self.m_DataObj.flagNoData = False
1352 self.m_DataObj.flagResetProcessing = self.flagResetProcessing
1353
1354 self.m_DataObj.data = self.datablock[self.__datablockIndex,:,:]
1355 self.m_DataObj.idProfile = self.idProfile
1356
1357 self.__datablockIndex += 1
1358 self.idProfile += 1
1359
1360 elif self.__format == "pdata":
1361 if self.__data_dc == None:
1362 self.m_Voltage.flagNoData = True
1363 return 0
1364
1365 self.m_DataObj.flagNoData = False
1366 self.m_DataObj.flagResetProcessing = self.flagResetProcessing
1367
1368 self.m_DataObj.data_spc = self.__data_spc
1369 self.m_DataObj.data_cspc = self.__data_cspc
1370 self.m_DataObj.data_dc = self.__data_dc
1371
1372 else:
1373 return 0
1374
1375 return 1 #self.m_Voltage.data
1376
1377
1378 class JRODataWriter(DataWriter):
1379 """
1380 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1381 de los datos siempre se realiza por bloques.
1382 """
1383
1384 def __init__(self):
1385 """
1386 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
1387
1388 Affected:
1389 self.m_Voltage
1390 self.m_BasicHeader
1391 self.m_SystemHeader
1392 self.m_RadarControllerHeader
1393 self.m_ProcessingHeader
1394
1395 Return: None
1396 """
1397 if m_Voltage == None:
1398 m_Voltage = Voltage()
1399
1400 self.m_Voltage = m_Voltage
1401
1402 self.__path = None
1403
1404 self.__fp = None
1405
1406 self.__format = None
1407
1408 self.__blocksCounter = 0
1409
1410 self.__setFile = None
1411
1412 self.__flagIsNewFile = 1
1413
1414 self.__dataType = None
1415
1416 self.datablock = None
1417
1418 self.__datablockIndex = 0
1419
1420 self.__ext = None
1421
1422 self.__shapeBuffer = None
1423
1424 self.__shape_spc_Buffer = None
1425 self.__shape_cspc_Buffer = None
1426 self.__shape_dc_Buffer = None
1427
1428 self.nWriteBlocks = 0
1429
1430 self.flagIsNewBlock = 0
1431
1432 self.noMoreFiles = 0
1433
1434 self.filename = None
1435
1436 self.m_BasicHeader= BasicHeader()
1437
1438 self.m_SystemHeader = SystemHeader()
1439
1440 self.m_RadarControllerHeader = RadarControllerHeader()
1441
1442 self.m_ProcessingHeader = ProcessingHeader()
1443
1444 self.__data_spc = None
1445 self.__data_cspc = None
1446 self.__data_dc = None
1447
1448
1449 def __writeFirstHeader(self):
1450 """
1451 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1452
1453 Affected:
1454 __dataType
1455
1456 Return:
1457 None
1458 """
1459 self.__writeBasicHeader()
1460 self.__wrSystemHeader()
1461 self.__wrRadarControllerHeader()
1462 self.__wrProcessingHeader()
1463 self.__dataType = self.m_DataObj.dataType
1464
1465
1466 def __writeBasicHeader(self, fp=None):
1467 """
1468 Escribe solo el Basic header en el file creado
1469
1470 Return:
1471 None
1472 """
1473 if fp == None:
1474 fp = self.__fp
1475
1476 self.m_BasicHeader.write(fp)
1477
1478
1479 def __wrSystemHeader(self, fp=None):
1480 """
1481 Escribe solo el System header en el file creado
1482
1483 Return:
1484 None
1485 """
1486 if fp == None:
1487 fp = self.__fp
1488
1489 self.m_SystemHeader.write(fp)
1490
1491
1492 def __wrRadarControllerHeader(self, fp=None):
1493 """
1494 Escribe solo el RadarController header en el file creado
1495
1496 Return:
1497 None
1498 """
1499 if fp == None:
1500 fp = self.__fp
1501
1502 self.m_RadarControllerHeader.write(fp)
1503
1504
1505 def __wrProcessingHeader(self, fp=None):
1506 """
1507 Escribe solo el Processing header en el file creado
1508
1509 Return:
1510 None
1511 """
1512 if fp == None:
1513 fp = self.__fp
1514
1515 self.m_ProcessingHeader.write(fp)
1516
1517
1518 def __setNextFile(self):
1519 """
1520 Determina el siguiente file que sera escrito
1521
1522 Affected:
1523 self.filename
1524 self.__subfolder
1525 self.__fp
1526 self.__setFile
1527 self.__flagIsNewFile
1528
1529 Return:
1530 0 : Si el archivo no puede ser escrito
1531 1 : Si el archivo esta listo para ser escrito
1532 """
1533 ext = self.__ext
1534 path = self.__path
1535
1536 if self.__fp != None:
1537 self.__fp.close()
1538
1539 timeTuple = time.localtime( self.m_DataObj.m_BasicHeader.utc ) # utc from m_Voltage
1540 subfolder = 'D%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1541
1542 tmp = os.path.join( path, subfolder )
1543 if not( os.path.exists(tmp) ):
1544 os.mkdir(tmp)
1545 self.__setFile = -1 #inicializo mi contador de seteo
1546 else:
1547 filesList = os.listdir( tmp )
1548 if len( filesList ) > 0:
1549 filesList = sorted( filesList, key=str.lower )
1550 filen = filesList[-1]
1551 # el filename debera tener el siguiente formato
1552 # 0 1234 567 89A BCDE (hex)
1553 # D YYYY DDD SSS .ext
1554 if isNumber( filen[8:11] ):
1555 self.__setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1556 else:
1557 self.__setFile = -1
1558 else:
1559 self.__setFile = -1 #inicializo mi contador de seteo
1560
1561 setFile = self.__setFile
1562 setFile += 1
1563
1564 if self.__format == "jicamarca":
1565 shead = "D"
1566 elif self.__format == "pdata":
1567 shead = "P"
1568
1569 file = '%s%4.4d%3.3d%3.3d%s' % ( shead, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1570
1571 filename = os.path.join( path, subfolder, file )
1572
1573 fp = open( filename,'wb' )
1574
1575 self.__blocksCounter = 0
1019 1576
1020 self.m_Voltage.flagNoData = False
1021 self.m_Voltage.flagNoContinuousBlock = self.flagNoContinuousBlock
1577 #guardando atributos
1578 self.filename = filename
1579 self.__subfolder = subfolder
1580 self.__fp = fp
1581 self.__setFile = setFile
1582 self.__flagIsNewFile = 1
1583
1584 print 'Writing the file: %s'%self.filename
1585
1586 self.__writeFirstHeader()
1587
1588 return 1
1589
1590
1591 def __setNewBlock(self):
1592 """
1593 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1594
1595 Return:
1596 0 : si no pudo escribir nada
1597 1 : Si escribio el Basic el First Header
1598 """
1599 if self.__fp == None:
1600 self.__setNextFile()
1601
1602 if self.__flagIsNewFile:
1603 return 1
1604
1605 if self.__blocksCounter < self.m_ProcessingHeader.dataBlocksPerFile:
1606 self.__writeBasicHeader()
1607 return 1
1608
1609 if not( self.__setNextFile() ):
1610 return 0
1611
1612 return 1
1613
1614
1615 def __writeVoltageBlock(self):
1616 """
1617 Escribe el buffer en el file designado
1618
1619 Affected:
1620 self.__datablockIndex
1621 self.__flagIsNewFile
1622 self.flagIsNewBlock
1623 self.nWriteBlocks
1624 self.__blocksCounter
1625
1626 Return: None
1627 """
1628 data = numpy.zeros( self.__shapeBuffer, self.__dataType )
1629
1630 data['real'] = self.datablock.real
1631 data['imag'] = self.datablock.imag
1632
1633 data = data.reshape( (-1) )
1634
1635 data.tofile( self.__fp )
1636
1637 self.datablock.fill(0)
1638 self.__datablockIndex = 0
1639 self.__flagIsNewFile = 0
1640 self.flagIsNewBlock = 1
1641 self.nWriteBlocks += 1
1642 self.__blocksCounter += 1
1643
1644
1645 def __writeSpectraBlock(self):
1646 """
1647 Escribe el buffer en el file designado
1648
1649 Affected:
1650 self.__data_spc
1651 self.__data_cspc
1652 self.__data_dc
1653 self.__flagIsNewFile
1654 self.flagIsNewBlock
1655 self.nWriteBlocks
1656 self.__blocksCounter
1657
1658 Return: None
1659 """
1660 spc = numpy.transpose( self.__data_spc, (0,2,1) )
1661 if not( self.m_ProcessingHeader.shif_fft ):
1662 spc = numpy.roll( spc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
1663 data = spc.reshape((-1))
1664 data.tofile(self.__fp)
1665
1666 data = numpy.zeros( self.__shape_cspc_Buffer, self.__dataType )
1667 cspc = numpy.transpose( self.__data_cspc, (0,2,1) )
1668 if not( self.m_ProcessingHeader.shif_fft ):
1669 cspc = numpy.roll( cspc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
1670 data['real'] = cspc.real
1671 data['imag'] = cspc.imag
1672 data = data.reshape((-1))
1673 data.tofile(self.__fp)
1674
1675 data = numpy.zeros( self.__shape_dc_Buffer, self.__dataType )
1676 dc = self.__data_dc
1677 data['real'] = dc.real
1678 data['imag'] = dc.imag
1679 data = data.reshape((-1))
1680 data.tofile(self.__fp)
1681
1682 self.__data_spc.fill(0)
1683 self.__data_cspc.fill(0)
1684 self.__data_dc.fill(0)
1685
1686 self.__flagIsNewFile = 0
1687 self.flagIsNewBlock = 1
1688 self.nWriteBlocks += 1
1689 self.__blocksCounter += 1
1690
1691
1692 def writeNextBlock(self):
1693 """
1694 Selecciona el bloque siguiente de datos y los escribe en un file
1695
1696 Return:
1697 0 : Si no hizo pudo escribir el bloque de datos
1698 1 : Si no pudo escribir el bloque de datos
1699 """
1700 if not( self.__setNewBlock() ):
1701 return 0
1702
1703 if self.__format == "jicamarca":
1704 self.__writeVoltageBlock()
1705
1706 if self.__format == "pdata":
1707 self.__writeSpectraBlock()
1708
1709 return 1
1710
1711
1712 def __hasAllDataInBuffer(self):
1713 if self.__format == "jicamarca":
1714 if self.__datablockIndex >= self.m_ProcessingHeader.profilesPerBlock:
1715 return 1
1716 else: return 0
1717
1718 return 1
1719
1720
1721 def putData(self):
1722 """
1723 Setea un bloque de datos y luego los escribe en un file
1724
1725 Affected:
1726 self.flagIsNewBlock
1727 self.__datablockIndex
1728
1729 Return:
1730 0 : Si no hay data o no hay mas files que puedan escribirse
1731 1 : Si se escribio la data de un bloque en un file
1732 """
1733 self.flagIsNewBlock = 0
1022 1734
1023 self.m_Voltage.data = self.datablock[self.__datablockIndex,:,:]
1024 self.m_Voltage.profileIndex = self.__datablockIndex
1735 if self.m_DataObj.flagNoData:
1736 return 0
1025 1737
1738 if self.m_DataObj.flagResetProcessing:
1739 if self.__format == "jicamarca":
1740 self.datablock.fill(0)
1741 self.__datablockIndex = 0
1742
1743 elif self.__format == "pdata":
1744 self.__data_spc.fill(0)
1745 self.__data_cspc.fill(0)
1746 self.__data_dc.fill(0)
1747
1748 self.__setNextFile()
1749
1750 if self.__format == "jicamarca":
1751 self.datablock[self.__datablockIndex,:,:] = self.m_Voltage.data
1752
1753 elif self.__format == "pdata":
1754 self.__data_spc = self.m_Spectra.data_spc
1755 self.__data_cspc = self.m_Spectra.data_cspc
1756 self.__data_dc = self.m_Spectra.data_dc
1757
1026 1758 self.__datablockIndex += 1
1027 1759
1028 #call setData - to Data Object
1029
1030 return self.m_Voltage.data
1031
1032
1033 #class VoltageWriter(DataWriter):
1034 # """
1035 # Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
1036 # de los datos siempre se realiza por bloques.
1037 # """
1038 # __configHeaderFile = 'wrSetHeadet.txt'
1039 #
1040 # def __init__( self, m_Voltage = None ):
1041 # """
1042 # Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
1043 #
1044 # Affected:
1045 # self.m_Voltage
1046 # self.m_BasicHeader
1047 # self.m_SystemHeader
1048 # self.m_RadarControllerHeader
1049 # self.m_ProcessingHeader
1050 #
1051 # Return: None
1052 # """
1053 # if m_Voltage == None:
1054 # m_Voltage = Voltage()
1055 #
1056 # self.m_Voltage = m_Voltage
1057 #
1058 # self.__path = None
1059 #
1060 # self.__fp = None
1061 #
1062 # self.__format = None
1063 #
1064 # self.__blocksCounter = 0
1065 #
1066 # self.__setFile = None
1067 #
1068 # self.__flagNewFile = 1
1069 #
1070 # self.datablock = None
1071 #
1072 # self.__datablockIndex = 0
1073 #
1074 # self.__dataType = None
1075 #
1076 # self.__ext = None
1077 #
1078 # self.__shapeBuffer = None
1079 #
1080 # self.nWriteBlocks = 0
1081 #
1082 # self.__flagNewBlock = 0
1083 #
1084 # self.flagNoMoreFiles = 0
1085 #
1086 # self.filename = None
1087 #
1088 # self.m_BasicHeader= BasicHeader()
1089 #
1090 # self.m_SystemHeader = SystemHeader()
1091 #
1092 # self.m_RadarControllerHeader = RadarControllerHeader()
1093 #
1094 # self.m_ProcessingHeader = ProcessingHeader()
1095 #
1096 #
1097 # def __writeFirstHeader( self ):
1098 # """
1099 # Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1100 #
1101 # Affected:
1102 # __dataType
1103 #
1104 # Return:
1105 # None
1106 # """
1107 # self.__writeBasicHeader()
1108 # self.__wrSystemHeader()
1109 # self.__wrRadarControllerHeader()
1110 # self.__wrProcessingHeader()
1111 # self.__dataType = self.m_Voltage.dataType
1112 #
1113 #
1114 # def __writeBasicHeader( self, fp=None ):
1115 # """
1116 # Escribe solo el Basic header en el file creado
1117 #
1118 # Return:
1119 # None
1120 # """
1121 # if fp == None:
1122 # fp = self.__fp
1123 #
1124 # self.m_BasicHeader.write(fp)
1125 #
1126 #
1127 # def __wrSystemHeader( self, fp=None ):
1128 # """
1129 # Escribe solo el System header en el file creado
1130 #
1131 # Return:
1132 # None
1133 # """
1134 # if fp == None:
1135 # fp = self.__fp
1136 #
1137 # self.m_SystemHeader.write(fp)
1138 #
1139 #
1140 # def __wrRadarControllerHeader( self, fp=None ):
1141 # """
1142 # Escribe solo el RadarController header en el file creado
1143 #
1144 # Return:
1145 # None
1146 # """
1147 # if fp == None:
1148 # fp = self.__fp
1149 #
1150 # self.m_RadarControllerHeader.write(fp)
1151 #
1152 #
1153 # def __wrProcessingHeader( self, fp=None ):
1154 # """
1155 # Escribe solo el Processing header en el file creado
1156 #
1157 # Return:
1158 # None
1159 # """
1160 # if fp == None:
1161 # fp = self.__fp
1162 #
1163 # self.m_ProcessingHeader.write(fp)
1164 #
1165 # def __setNextFile( self ):
1166 # """
1167 # Determina el siguiente file que sera escrito
1168 #
1169 # Affected:
1170 # self.filename
1171 # self.__subfolder
1172 # self.__fp
1173 # self.__setFile
1174 # self.__flagNewFile
1175 #
1176 # Return:
1177 # 0 : Si el archivo no puede ser escrito
1178 # 1 : Si el archivo esta listo para ser escrito
1179 # """
1180 # #setFile = self.__setFile
1181 # ext = self.__ext
1182 # path = self.__path
1183 #
1184 # #setFile += 1
1185 #
1186 # if self.__fp != None:
1187 # self.__fp.close()
1188 #
1189 # """
1190 # timeTuple = time.localtime(self.m_Voltage.m_BasicHeader.utc) # utc from m_Voltage
1191 # file = 'D%4.4d%3.3d%3.3d%s' % (timeTuple.tm_year,timeTuple.tm_yday,setFile,ext)
1192 # subfolder = 'D%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1193 # tmp = os.path.join(path,subfolder)
1194 # if not(os.path.exists(tmp)):
1195 # os.mkdir(tmp)
1196 # """
1197 # ##################################
1198 # if self.m_BasicHeader.size <= 24: return 0 #no existe la suficiente data para ser escrita
1199 #
1200 # timeTuple = time.localtime( self.m_Voltage.m_BasicHeader.utc ) # utc from m_Voltage
1201 # subfolder = 'D%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1202 #
1203 # tmp = os.path.join( path, subfolder )
1204 # if not( os.path.exists(tmp) ):
1205 # os.mkdir(tmp)
1206 # self.__setFile = -1 #inicializo mi contador de seteo
1207 # else:
1208 # filesList = os.listdir( tmp )
1209 # if len( filesList ) > 0:
1210 # filesList = sorted( filesList, key=str.lower )
1211 # filen = filesList[-1]
1212 # # el filename debera tener el siguiente formato
1213 # # 0 1234 567 89A BCDE (hex)
1214 # # D YYYY DDD SSS .ext
1215 # if isNumber( filen[8:11] ):
1216 # self.__setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1217 # else:
1218 # self.__setFile = -1
1219 # else:
1220 # self.__setFile = -1 #inicializo mi contador de seteo
1221 #
1222 # setFile = self.__setFile
1223 # setFile += 1
1224 # file = 'D%4.4d%3.3d%3.3d%s' % ( timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1225 # ##################################
1226 #
1227 # filename = os.path.join( path, subfolder, file )
1228 #
1229 # fp = open( filename,'wb' )
1230 #
1231 # self.__blocksCounter = 0
1232 #
1233 # #guardando atributos
1234 # self.filename = filename
1235 # self.__subfolder = subfolder
1236 # self.__fp = fp
1237 # self.__setFile = setFile
1238 # self.__flagNewFile = 1
1239 #
1240 # print 'Writing the file: %s'%self.filename
1241 #
1242 # self.__writeFirstHeader()
1243 #
1244 # return 1
1245 #
1246 #
1247 # def __setNewBlock( self ):
1248 # """
1249 # Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1250 #
1251 # Return:
1252 # 0 : si no pudo escribir nada
1253 # 1 : Si escribio el Basic el First Header
1254 # """
1255 # if self.__fp == None:
1256 # self.__setNextFile()
1257 #
1258 # if self.__flagNewFile:
1259 # return 1
1260 #
1261 # if self.__blocksCounter < self.m_ProcessingHeader.dataBlocksPerFile:
1262 # self.__writeBasicHeader()
1263 # return 1
1264 #
1265 # if not( self.__setNextFile() ):
1266 # return 0
1267 #
1268 # return 1
1269 #
1270 # def __writeBlock( self ):
1271 # """
1272 # Escribe el buffer en el file designado
1273 #
1274 # Affected:
1275 # self.__datablockIndex
1276 # self.__flagNewFile
1277 # self.__flagNewBlock
1278 # self.nWriteBlocks
1279 # self.__blocksCounter
1280 #
1281 # Return: None
1282 # """
1283 # data = numpy.zeros( self.__shapeBuffer, self.__dataType )
1284 #
1285 # data['real'] = self.datablock.real
1286 # data['imag'] = self.datablock.imag
1287 #
1288 # data = data.reshape( (-1) )
1289 #
1290 # data.tofile( self.__fp )
1291 #
1292 # self.datablock.fill(0)
1293 #
1294 # self.__datablockIndex = 0
1295 #
1296 # self.__flagNewFile = 0
1297 #
1298 # self.__flagNewBlock = 1
1299 #
1300 # self.nWriteBlocks += 1
1301 #
1302 # self.__blocksCounter += 1
1303 #
1304 #
1305 # def writeNextBlock( self ):
1306 # """
1307 # Selecciona el bloque siguiente de datos y los escribe en un file
1308 #
1309 # Return:
1310 # 0 : Si no hizo pudo escribir el bloque de datos
1311 # 1 : Si no pudo escribir el bloque de datos
1312 # """
1313 # if not(self.__setNewBlock()):
1314 # return 0
1315 #
1316 # self.__writeBlock()
1317 #
1318 # return 1
1319 #
1320 #
1321 # def __hasAllDataInBuffer( self ):
1322 # if self.__datablockIndex >= self.m_ProcessingHeader.profilesPerBlock:
1323 # return 1
1324 #
1325 # return 0
1326 #
1327 #
1328 # def putData( self ):
1329 # """
1330 # Setea un bloque de datos y luego los escribe en un file
1331 #
1332 # Affected:
1333 # self.__flagNewBlock
1334 # self.__datablockIndex
1335 #
1336 # Return:
1337 # 0 : Si no hay data o no hay mas files que puedan escribirse
1338 # 1 : Si se escribio la data de un bloque en un file
1339 # """
1340 # self.__flagNewBlock = 0
1341 #
1342 # if self.m_Voltage.flagNoData:
1343 # return 0
1344 #
1345 # if self.m_Voltage.flagNoContinuousBlock:
1346 #
1347 # self.datablock.fill(0)
1348 #
1349 # self.__datablockIndex = 0
1350 # self.__setNextFile()
1351 #
1352 # self.datablock[self.__datablockIndex,:,:] = self.m_Voltage.data
1353 #
1354 # self.__datablockIndex += 1
1355 #
1356 # if self.__hasAllDataInBuffer():
1357 #
1358 # self.__getHeader()
1359 # self.writeNextBlock()
1360 #
1361 # if self.flagNoMoreFiles:
1362 # #print 'Process finished'
1363 # return 0
1364 #
1365 # return 1
1366 #
1367 #
1368 # def __getHeader( self ):
1369 # """
1370 # Obtiene una copia del First Header
1371 #
1372 # Affected:
1373 # self.m_BasicHeader
1374 # self.m_SystemHeader
1375 # self.m_RadarControllerHeader
1376 # self.m_ProcessingHeader
1377 # self.__dataType
1378 #
1379 # Return:
1380 # None
1381 # """
1382 # self.m_BasicHeader = self.m_Voltage.m_BasicHeader.copy()
1383 # self.m_SystemHeader = self.m_Voltage.m_SystemHeader.copy()
1384 # self.m_RadarControllerHeader = self.m_Voltage.m_RadarControllerHeader.copy()
1385 # self.m_ProcessingHeader = self.m_Voltage.m_ProcessingHeader.copy()
1386 # self.__dataType = self.m_Voltage.dataType
1387 #
1388 #
1389 # def __setHeaderByFile( self ):
1390 #
1391 # format = self.__format
1392 # header = ['Basic','System','RadarController','Processing']
1393 #
1394 # fmtFromFile = None
1395 # headerFromFile = None
1396 #
1397 #
1398 # fileTable = self.__configHeaderFile
1399 #
1400 # if os.access(fileTable, os.R_OK):
1401 # import re, string
1402 #
1403 # f = open(fileTable,'r')
1404 # lines = f.read()
1405 # f.close()
1406 #
1407 # #Delete comments into expConfig
1408 # while 1:
1409 #
1410 # startComment = string.find(lines.lower(),'#')
1411 # if startComment == -1:
1412 # break
1413 # endComment = string.find(lines.lower(),'\n',startComment)
1414 # lines = string.replace(lines,lines[startComment:endComment+1],'', 1)
1415 #
1416 # while expFromFile == None:
1417 #
1418 # currFmt = string.find(lines.lower(),'format="%s"' %(expName))
1419 # nextFmt = string.find(lines.lower(),'format',currFmt+10)
1420 #
1421 # if currFmt == -1:
1422 # break
1423 # if nextFmt == -1:
1424 # nextFmt = len(lines)-1
1425 #
1426 # fmtTable = lines[currFmt:nextFmt]
1427 # lines = lines[nextFmt:]
1428 #
1429 # fmtRead = self.__getValueFromArg(fmtTable,'format')
1430 # if fmtRead != format:
1431 # continue
1432 # fmtFromFile = fmtRead
1433 #
1434 # lines2 = fmtTable
1435 #
1436 # while headerFromFile == None:
1437 #
1438 # currHeader = string.find(lines2.lower(),'header="%s"' %(header))
1439 # nextHeader = string.find(lines2.lower(),'header',currHeader+10)
1440 #
1441 # if currHeader == -1:
1442 # break
1443 # if nextHeader == -1:
1444 # nextHeader = len(lines2)-1
1445 #
1446 # headerTable = lines2[currHeader:nextHeader]
1447 # lines2 = lines2[nextHeader:]
1448 #
1449 # headerRead = self.__getValueFromArg(headerTable,'site')
1450 # if not(headerRead in header):
1451 # continue
1452 # headerFromFile = headerRead
1453 #
1454 # if headerRead == 'Basic':
1455 # self.m_BasicHeader.size = self.__getValueFromArg(headerTable,'size',lower=False)
1456 # self.m_BasicHeader.version = self.__getValueFromArg(headerTable,'version',lower=False)
1457 # self.m_BasicHeader.dataBlock = self.__getValueFromArg(headerTable,'dataBlock',lower=False)
1458 # self.m_BasicHeader.utc = self.__getValueFromArg(headerTable,'utc',lower=False)
1459 # self.m_BasicHeader.miliSecond = self.__getValueFromArg(headerTable,'miliSecond',lower=False)
1460 # self.m_BasicHeader.timeZone = self.__getValueFromArg(headerTable,'timeZone',lower=False)
1461 # self.m_BasicHeader.dstFlag = self.__getValueFromArg(headerTable,'dstFlag',lower=False)
1462 # self.m_BasicHeader.errorCount = self.__getValueFromArg(headerTable,'errorCount',lower=False)
1463 #
1464 # else:
1465 # print "file access denied:%s"%fileTable
1466 # sys.exit(0)
1467 #
1468 #
1469 # def setup( self, path, set=0, format='rawdata' ):
1470 # """
1471 # Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1472 #
1473 # Inputs:
1474 # path : el path destino en el cual se escribiran los files a crear
1475 # format : formato en el cual sera salvado un file
1476 # set : el setebo del file
1477 #
1478 # Return:
1479 # 0 : Si no realizo un buen seteo
1480 # 1 : Si realizo un buen seteo
1481 # """
1482 # if format == 'hdf5':
1483 # ext = '.hdf5'
1484 # format = 'hdf5'
1485 # print 'call hdf5 library'
1486 # return 0
1487 #
1488 # if format == 'rawdata':
1489 # ext = '.r'
1490 # format = 'Jicamarca'
1491 #
1492 # #call to config_headers
1493 # #self.__setHeaderByFile()
1494 #
1495 # self.__path = path
1496 # self.__setFile = set - 1
1497 # self.__ext = ext
1498 # self.__format = format
1499 #
1500 # self.__getHeader()
1501 # self.__shapeBuffer = (self.m_ProcessingHeader.profilesPerBlock,
1502 # self.m_ProcessingHeader.numHeights,
1503 # self.m_SystemHeader.numChannels )
1504 #
1505 # self.datablock = numpy.zeros(self.__shapeBuffer, numpy.dtype('complex'))
1506 #
1507 ## if not(self.__setNextFile()):
1508 ## return 0
1509 # return 1
1510
1511 class JRODataWriter():
1760 if self.__hasAllDataInBuffer():
1761 self.__getHeader()
1762 self.writeNextBlock()
1763
1764 if self.noMoreFiles:
1765 #print 'Process finished'
1766 return 0
1767
1768 return 1
1769
1770
1771 def __getHeader(self):
1772 """
1773 Obtiene una copia del First Header
1774
1775 Affected:
1776 self.m_BasicHeader
1777 self.m_SystemHeader
1778 self.m_RadarControllerHeader
1779 self.m_ProcessingHeader
1780 self.__dataType
1781
1782 Return:
1783 None
1784 """
1785 self.m_BasicHeader = self.m_DataObj.m_BasicHeader.copy()
1786 self.m_SystemHeader = self.m_DataObj.m_SystemHeader.copy()
1787 self.m_RadarControllerHeader = self.m_DataObj.m_RadarControllerHeader.copy()
1788 self.m_ProcessingHeader = self.m_DataObj.m_ProcessingHeader.copy()
1789 self.__dataType = self.m_DataObj.dataType
1790
1791
1792 def setup(self, path, set=0, ext='.pdata'):
1793 """
1794 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1795
1796 Inputs:
1797 path : el path destino en el cual se escribiran los files a crear
1798 format : formato en el cual sera salvado un file
1799 set : el setebo del file
1800
1801 Return:
1802 0 : Si no realizo un buen seteo
1803 1 : Si realizo un buen seteo
1804 """
1805
1806 ext = ext.lower()
1807
1808 if ext == '.hdf5':
1809 print 'call hdf5 library'
1810 return 0
1811
1812 elif ext == '.r':
1813 format = 'jicamarca'
1814
1815 elif ext == '.pdata':
1816 format = 'pdata'
1817
1818 else:
1819 print 'unknow format !!!'
1820 return 0
1821
1822 self.__path = path
1823 self.__setFile = set - 1
1824 self.__ext = ext
1825 self.__format = format
1826 self.__getHeader()
1827
1828 if self.__format == "jicamarca":
1829 self.__shapeBuffer = (self.m_ProcessingHeader.profilesPerBlock,
1830 self.m_ProcessingHeader.numHeights,
1831 self.m_SystemHeader.numChannels )
1832
1833 self.datablock = numpy.zeros(self.__shapeBuffer, numpy.dtype('complex'))
1834
1835 elif self.__format == "pdata":
1836 self.__shape_spc_Buffer = ( self.m_Spectra.nChannels,
1837 self.m_ProcessingHeader.numHeights,
1838 self.m_ProcessingHeader.profilesPerBlock
1839 )
1512 1840
1513 def __init__(self):
1514 pass No newline at end of file
1841 self.__shape_cspc_Buffer = ( self.m_Spectra.nPairs,
1842 self.m_ProcessingHeader.numHeights,
1843 self.m_ProcessingHeader.profilesPerBlock
1844 )
1845
1846 self.__shape_dc_Buffer = ( self.m_SystemHeader.numChannels,
1847 self.m_ProcessingHeader.numHeights
1848 )
1849
1850 if not( self.__setNextFile() ):
1851 print "There isn't a next file"
1852 return 0
1853
1854 return 1
General Comments 0
You need to be logged in to leave comments. Login now