schain-jc Commit - r47:580af48a6d5c · Jicamarca Repository

SPECTRAIO.py:...

Victor Sarmiento -

r47:580af48a6d5c

parent child

Context file:

r47:580af48a6d5c

Collapse all files

schainpy/IO/SpectraIO.py +177 -90

             from Model.Spectra import Spectra
+            def isFileOK(filename):
+                """
+                Determina si la cabecera de un archivo es valido o no, si lo es entonces seria un archivo que podria contener data,
+                si no seria un archivo invalido
+                Return:
+                    True    :    si es un archivo valido
+                    False   :    si no es un archivo valido
+                Exceptions:
+                    Si al leer la cabecera esta no coincide con el tipo de las variables que la contienen entonces se dispara
+                    una exception
+                """
+                m_BasicHeader = BasicHeader()
+                m_ProcessingHeader = ProcessingHeader()
+                m_RadarControllerHeader = RadarControllerHeader()
+                m_SystemHeader = SystemHeader()
+                fp = None
+                try:
+                    fp = open( filename,'rb' ) #lectura binaria
+                    m_BasicHeader.read(fp)
+                    m_SystemHeader.read(fp)
+                    m_RadarControllerHeader.read(fp)
+                    m_ProcessingHeader.read(fp)
+                    fp.close()
+                except:
+                    if fp != None: fp.close()
+                    return False
+                return True
             def getlastFileFromPath(pathList,ext):
                 """
                 Depura el pathList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
                     self.datablock_id = 9999
-                    self.__delay  = 7   #seconds
+                    self.__delay  = 2   #seconds
                     self.__nTries  = 3  #quantity tries
                     self.__nFiles = 3   #number of files for searching
                     self.__year = 0
                     self.__set = 0
                     self.__ext = None
                     self.__path = None
+                    self.nBlocks = 0
                 def __rdSystemHeader(self,fp=None):
                     """
                     self.__blocksize = self.__pts2read_SelfSpectra + self.__pts2read_CrossSpectra + self.__pts2read_DCchannels
-                    print "SIZEEEE ",self.__blocksize, self.m_ProcessingHeader.blockSize
                     self.m_Spectra.nChannels = self.nChannels
                     self.m_Spectra.nPairs = self.nPairs
                     countFiles = 0
                     countTries = 0
-                    fileStatus = 0
+                    #fileStatus = 0
                     notFirstTime_flag = False
-                    bChangeDir = False
+                    fileOk_flag = False
+                    changeDir_flag = False
                     fileSize = 0
                     fp = None
                         if countFiles > self.__nFiles: #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
                             self.__set = 0
                             self.__doy += 1
-                            bChangeDir = True
+                            changeDir_flag = True
                         file = None
                         filename = None
+                        fileOk_flag = False
+                        #busca el 1er file disponible
+                        file, filename = checkForRealPath( self.__path, self.__year, self.__doy, self.__set, self.__ext )
+                        if file == None:
+                            if notFirstTime_flag: #si no es la primera vez que busca el file entonces no espera y busca for el siguiente file
+                                print "\tsearching next \"%s\" file ..." % ( filename )
+                                continue
+                            else: #si es la primera vez que busca el file entonces espera self.__nTries veces hasta encontrarlo o no
+                                for nTries in range( self.__nTries ):
+                                    print "\twaiting new \"%s\" file, try %03d ..." % ( filename, nTries+1 )
+                                    time.sleep( self.__delay )
+                                    file, filename = checkForRealPath( self.__path, self.__year, self.__doy, self.__set, self.__ext )
+                                    if file != None:
+                                        fileOk_flag = True
+                                        break
+                                if not( fileOk_flag ): #no encontro ningun file valido a leer
+                                    notFirstTime_flag = True
+                                    continue
-                        countTries = 0
+                        """countTries = 0
                         #espero hasta encontrar el 1er file disponible
                         while( True ):
                         if countTries >= self.__nTries: #se realizaron n intentos y no hubo un file nuevo
                             notFirstTime_flag = True
                             continue #vuelvo al inico del while principal
+                        """
-                        countTries = 0
+                        #una vez que se obtuvo el 1er file valido se procede a checkear si si tamanho es suficiente para empezar a leerlo
+                        currentSize = os.path.getsize( file )
-                        #una vez que se obtuvo el 1er file valido se procede a checkear su contenido, y se espera una cierta cantidad
-                        #de tiempo por una cierta cantidad de veces hasta que el contenido del file sea un contenido valido
-                        while( True ):
-                            countTries += 1
-                            if countTries > self.__nTries:
-                                break
-                            try:
-                                fp = open(file)
-                            except:
-                                print "The file \"%s\" can't be opened" % file
-                                break
-                            fileSize = os.path.getsize( file )
-                            currentSize = fileSize - fp.tell()
                         neededSize = self.m_ProcessingHeader.blockSize + self.firstHeaderSize
+                        #si el tamanho es suficiente entonces deja de buscar
                         if currentSize > neededSize:
-                                fileStatus = 1
+                            fileOk_flag = True
                             break
-                            fp.close()
+                        fileOk_flag = False
+                        #si el file no tiene el tamanho necesario se espera una cierta cantidad de tiempo
-                            if bChangeDir: #si al buscar un file cambie de directorio ya no espero y salgo del bucle while
+                        #por una cierta cantidad de veces hasta que el contenido del file sea valido
+                        if changeDir_flag: #si al buscar un file cambie de directorio ya no espero y sigo con el siguiente file
                             print "\tsearching next \"%s\" file ..." % filename
-                                break
+                            changeDir_flag = False
+                            continue
-                            print "\twaiting for block of \"%s\" file ..." % filename
+                        for nTries in range( self.__nTries ):
+                            print "\twaiting for the First Header block of \"%s\" file, try %03d ..." % ( filename, nTries+1 )
                             time.sleep( self.__delay )
-                        if fileStatus == 1:
+                            currentSize = os.path.getsize( file )
+                            neededSize = self.m_ProcessingHeader.blockSize + self.firstHeaderSize
+                            if currentSize > neededSize:
+                                fileOk_flag = True
+                                break
+                        if fileOk_flag: #si encontro un file valido sale del bucle y deja de buscar
                             break
                         print "Skipping the file \"%s\" due to this files is empty" % filename
                         countFiles = 0
+                    if fileOk_flag:
-                    if fileStatus == 1:
+                        self.fileSize = os.path.getsize( file ) #fileSize
-                        self.fileSize = fileSize
                         self.filename = file#name
                         self.__flagIsNewFile = 1
-                        self.__fp = fp
+                        self.__fp = open(file)
                         self.noMoreFiles = 0
                         print 'Setting the file: %s' % file #name
                     else:
                         self.noMoreFiles = 1
                         print 'No more Files'
-                    return fileStatus
+                    return fileOk_flag
                 def __setNextFileOffline( self ):
                         if not( idFile < len(self.filenameList) ):
                             self.noMoreFiles = 1
+                            print 'No more Files'
                             return 0
                         filename = self.filenameList[idFile]
                             continue
                         break
                     self.__flagIsNewFile = 1
                     self.__idFile = idFile
                     self.filename = filename
                     else:
                         newFile = self.__setNextFileOffline()
+                    if self.noMoreFiles:
+                        sys.exit(0)
                     if not(newFile):
                         return 0
                     self.__readFirstHeader()
+                    self.nBlocks = 0
                     return 1
                     if ( currentSize >= neededSize ):
                         self.__rdBasicHeader()
                         return 1
-                    elif self.online:
-                        nTries = 0
+                    #si es OnLine y ademas aun no se han leido un bloque completo entonces se espera por uno valido
-                        while( nTries < self.__nTries ):
+                    elif (self.nBlocks != self.m_ProcessingHeader.dataBlocksPerFile) and self.online:
-                            nTries += 1
+                        for nTries in range( self.__nTries ):
-                            print "Waiting for the next block, try %03d ..." % nTries
+                            fpointer = self.__fp.tell()
+                            self.__fp.close()
+                            print "\tWaiting for the next block, try %03d ..." % (nTries+1)
                             time.sleep( self.__delay )
-                            fileSize = os.path.getsize(self.filename)
+                            self.__fp = open( self.filename, 'rb' )
-                            currentSize = fileSize - self.__fp.tell()
+                            self.__fp.seek( fpointer )
+                            self.fileSize = os.path.getsize( self.filename )
+                            currentSize = self.fileSize - self.__fp.tell()
                             neededSize = self.m_ProcessingHeader.blockSize + self.basicHeaderSize
                             if ( currentSize >= neededSize ):
                     if deltaTime > self.__maxTimeStep:
                         self.flagResetProcessing = 1
-                        self.nReadBlocks = 0
+                        #self.nReadBlocks = 0
                     return 1
                         self.__data_spc
                         self.__data_cspc
                         self.__data_dc
+                    Exceptions:
+                        Si un bloque leido no es un bloque valido
                     """
-                    self.datablock_id = 0
+                    #self.datablock_id = 0
-                    self.__flagIsNewFile = 0
+                    #self.__flagIsNewFile = 0
-                    self.flagIsNewBlock = 1
+                    #self.flagIsNewBlock = 1
+                    blockOk_flag = False
                     fpointer = self.__fp.tell()
                     spc = numpy.fromfile( self.__fp, self.__dataType[0], self.__pts2read_SelfSpectra )
                     if self.online:
                         if (spc.size + cspc.size + dc.size) != self.__blocksize:
-                            nTries = 0
+                            for nTries in range( self.__nTries ):
-                            while( nTries < self.__nTries ):
+                                #nTries = 0
-                                nTries += 1
+                                #while( nTries < self.__nTries ):
-                                print "Waiting for the next block, try %03d ..." % nTries
+                                #nTries += 1
+                                print "\tWaiting for the next block, try %03d ..." % (nTries+1)
                                 time.sleep( self.__delay )
                                 self.__fp.seek( fpointer )
                                 fpointer = self.__fp.tell()
                                 spc = numpy.fromfile( self.__fp, self.__dataType[0], self.__pts2read_SelfSpectra )
                                 cspc = numpy.fromfile( self.__fp, self.__dataType, self.__pts2read_CrossSpectra )
                                 dc = numpy.fromfile( self.__fp, self.__dataType, self.__pts2read_DCchannels ) #int(self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels) )
                                 if (spc.size + cspc.size + dc.size) == self.__blocksize:
-                                    nTries = 0
+                                    blockOk_flag = True
                                     break
-                            if nTries > 0:
+                                #if (spc.size + cspc.size + dc.size) == self.__blocksize:
-                                return
+                                #    nTries = 0
+                                #    break
+                            if not( blockOk_flag ):
+                                return 0
+                            #if nTries > 0:
+                            #    return 0
+                    try:
                         spc = spc.reshape( (self.nChannels, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D
                         cspc = cspc.reshape( (self.nPairs, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D
                         dc = dc.reshape( (self.m_SystemHeader.numChannels, self.m_ProcessingHeader.numHeights) ) #transforma a un arreglo 2D
+                    except:
+                        print "Data file %s is invalid" % self.filename
+                        return 0
                     if not( self.m_ProcessingHeader.shif_fft ):
                         spc = numpy.roll( spc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                     self.__data_cspc = cspc['real'] + cspc['imag']*1j
                     self.__data_dc = dc['real'] + dc['imag']*1j
+                    self.datablock_id = 0
                     self.__flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nReadBlocks += 1
-                    self.datablock_id = 0
+                    self.nBlocks += 1
+                    return 1
                 def __hasNotDataInBuffer(self):
                         filename    :    el ultimo file de una determinada carpeta
                         directory   :    eL directorio donde esta el file encontrado
                     """
                     print "Searching files ..."
                     dirList = []
                         dirList = sorted( dirList, key=str.lower ) #para que quede ordenado al margen de si el nombre esta en mayusculas o minusculas, utilizo la funcion sorted
                         if len(dirList) > 0 :
-                            directory = dirList[-1]
+                            directory = dirList[-1] #me quedo con el ultimo directorio de una carpeta
                     else:
                         year = startDateTime.timetuple().tm_year
                         doy = startDateTime.timetuple().tm_yday
                                               de leer todos los bloques de datos de un determinado
                                               archivo se pasa al siguiente archivo de la lista.
                     """
                     print "Searching files ..."
                     dirList = []
                     file = os.path.join( path, dirfilename, filename )
-                    nTries = 0
+                    for nTries in range( self.__nTries+1 ):
-                    while(True):
-                        nTries += 1
-                        if nTries > self.__nTries:
-                            break
                         try:
                             fp = open( file,'rb' ) #lectura binaria
                         except:
                         if m_BasicHeader.size > 24:
                             break
-                        print 'waiting for new block: try %02d' % ( nTries )
+                        if nTries >= self.__nTries: #si ya espero la cantidad de veces necesarias entonces ya no vuelve a esperar
+                            break
+                        print '\twaiting for new block of file %s: try %02d' % ( file, nTries )
                         time.sleep( self.__delay)
                     if m_BasicHeader.size <= 24:
                         self.filenameList
                         self.online
                     """
                     if online:
-                        nTries = 0
+                        fileOK_flag = False
-                        while( nTries < self.__nTries ):
-                           nTries += 1
                         subfolder = "D%04d%03d" % ( startDateTime.timetuple().tm_year, startDateTime.timetuple().tm_yday )
-                           year, doy, set, filename, dirfilename = self.__searchFilesOnLine( path, startDateTime, ext )
-                           if filename == None:
                         file = os.path.join( path, subfolder )
-                               print "Searching first file in \"%s\", try %03d ..." % ( file, nTries )
-                               time.sleep( self.__delay )
+                        for nTries in range( self.__nTries+1 ): #espera por el 1er file
-                           else:
+                           year, doy, set, filename, dirfilename = self.__searchFilesOnLine( path, startDateTime, ext )
+                           if filename != None:
+                               if isFileOK( os.path.join( path,dirfilename,filename ) ):
+                                   fileOK_flag = True
                                    break
-                        if filename == None:
+                           if nTries >= self.__nTries: #si ya espero la cantidad de veces necesarias entonces ya no vuelve a esperar
-                            print "No files  On Line"
+                               break
+                           print "Searching first file in \"%s\", try %03d ..." % ( file, nTries+1 )
+                           time.sleep( self.__delay )
+                        if not( fileOK_flag ): #filename == None:
+                            print "No files  on line or invalid first file"
                             return 0
                         if self.__initFilesOnline( path, dirfilename, filename ) == 0:
-                            print "The file %s hasn't enough data"
+                            print "The file %s hasn't enough data" % filename
                             return 0
                         self.__year = year
                     if not( self.__setNewBlock() ):
                         return 0
-                    self.__readBlock()
+                    if not( self.__readBlock() ):
+                        return 0
                     self.__lastUTTime = self.m_BasicHeader.utc
                     if self.__hasNotDataInBuffer():
-                        self.readNextBlock()
+                        if not( self.readNextBlock() ):
+                            self.__setNextFile()
+                            return 0
                         self.m_Spectra.m_BasicHeader = self.m_BasicHeader.copy()
                         self.m_Spectra.m_ProcessingHeader = self.m_ProcessingHeader.copy()
                     #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
+                    if self.__data_dc == None:
+                        self.m_Voltage.flagNoData = True
+                        return 0
                     self.m_Spectra.flagNoData = False
                     self.m_Spectra.flagResetProcessing = self.flagResetProcessing
                     self.__wrProcessingHeader()
                     self.__dataType = self.m_Spectra.dataType
                 def __writeBasicHeader(self, fp=None):
                     """
                     Escribe solo el Basic header en el file creado
                     self.m_BasicHeader.write(fp)
                 def __wrSystemHeader(self,fp=None):
                     """
                     Escribe solo el System header en el file creado
                     self.m_SystemHeader.write(fp)
                 def __wrRadarControllerHeader(self,fp=None):
                     """
                     Escribe solo el RadarController header en el file creado
                     self.m_RadarControllerHeader.write(fp)
                 def __wrProcessingHeader(self,fp=None):
                     """
                     Escribe solo el Processing header en el file creado
                     self.m_ProcessingHeader.write(fp)
                 def __setNextFile(self):
                     """
                     Determina el siguiente file que sera escrito
                     if self.__fp != None:
                         self.__fp.close()
-                    if self.m_BasicHeader.size <= 24: return 0 #no existe la suficiente data para ser escrita
+                    #if self.m_BasicHeader.size <= 24: return 0 #no existe la suficiente data para ser escrita
                     timeTuple = time.localtime(self.m_Spectra.m_BasicHeader.utc) # utc from m_Spectra
                     subfolder = 'D%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     return 1
                 def __setNewBlock(self):
                     """
                     Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
                     return 1
                 def __writeBlock(self):
                     """
                     Escribe el buffer en el file designado
                     self.__data_dc.fill(0)
                     self.__flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nWriteBlocks += 1
                     self.__blocksCounter += 1
                         return 0
                     self.__writeBlock()
                     return 1
                     self.__data_dc = self.m_Spectra.data_dc
                     if True:
-                        #time.sleep( 3 )
                         self.__getHeader()
                         self.writeNextBlock()

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages