'''
Created on Jul 2, 2014

@author: roj-idl71
'''
import os
import sys
import glob
import time
import numpy
import fnmatch
import time, datetime
#import h5py
import traceback

try:
    from gevent import sleep
except:
    from time import sleep
    
from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width

LOCALTIME = True

def isNumber(cad):
    """
    Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.

    Excepciones: 
        Si un determinado string no puede ser convertido a numero
    Input:
        str, string al cual se le analiza para determinar si convertible a un numero o no
        
    Return:
        True    :    si el string es uno numerico
        False   :    no es un string numerico
    """
    try:
        float( cad )
        return True
    except:
        return False

def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
    """
    Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
    
    Inputs:
        filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
        
        startUTSeconds      :    fecha inicial del rango seleccionado. La fecha esta dada en
                                 segundos contados desde 01/01/1970.
        endUTSeconds        :    fecha final del rango seleccionado. La fecha esta dada en
                                 segundos contados desde 01/01/1970.
    
    Return:
        Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                        fecha especificado, de lo contrario retorna False.
    
    Excepciones:
        Si el archivo no existe o no puede ser abierto
        Si la cabecera no puede ser leida.
        
    """
    basicHeaderObj = BasicHeader(LOCALTIME)
    
    try:
        fp = open(filename,'rb')
    except IOError:
        print "The file %s can't be opened" %(filename)
        return 0
    
    sts = basicHeaderObj.read(fp)
    fp.close()
    
    if not(sts):
        print "Skipping the file %s because it has not a valid header" %(filename)
        return 0
    
    if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
        return 0
    
    return 1

def isTimeInRange(thisTime, startTime, endTime):
    
    if endTime >= startTime:
        if (thisTime < startTime) or (thisTime > endTime):
            return 0
        
        return 1
    else:
        if (thisTime < startTime) and (thisTime > endTime):
            return 0
        
        return 1
                
def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
    """
    Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
    
    Inputs:
        filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
        
        startDate          :    fecha inicial del rango seleccionado en formato datetime.date
        
        endDate            :    fecha final del rango seleccionado en formato datetime.date
        
        startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
        
        endTime            :    tiempo final del rango seleccionado en formato datetime.time
    
    Return:
        Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                        fecha especificado, de lo contrario retorna False.
    
    Excepciones:
        Si el archivo no existe o no puede ser abierto
        Si la cabecera no puede ser leida.
        
    """
    
    
    try:
        fp = open(filename,'rb')
    except IOError:
        print "The file %s can't be opened" %(filename)
        return None
    
    firstBasicHeaderObj = BasicHeader(LOCALTIME)
    systemHeaderObj = SystemHeader()
    radarControllerHeaderObj = RadarControllerHeader()
    processingHeaderObj = ProcessingHeader()
    
    lastBasicHeaderObj = BasicHeader(LOCALTIME)
    
    sts = firstBasicHeaderObj.read(fp)
    
    if not(sts):
        print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
        return None
    
    sts = systemHeaderObj.read(fp)
    sts = radarControllerHeaderObj.read(fp)
    sts = processingHeaderObj.read(fp)
    
    filesize = os.path.getsize(filename)
    
    offset = processingHeaderObj.blockSize + 24 #header size
    
    if filesize <= offset:
        print "[Reading] %s: This file has not enough data" %filename
        return None
    
    fp.seek(-offset, 2)
    
    sts = lastBasicHeaderObj.read(fp)
    
    fp.close()
    
    thisDatetime = lastBasicHeaderObj.datatime
    thisTime_last_block = thisDatetime.time()
    
    thisDatetime = firstBasicHeaderObj.datatime
    thisDate = thisDatetime.date()
    thisTime_first_block = thisDatetime.time()
    
    #General case
    #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
    #-----------o----------------------------o-----------
    #       startTime                     endTime
    
    if endTime >= startTime:
        if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
            return None
        
        return thisDatetime
    
    #If endTime < startTime then endTime belongs to the next day
    
    
    #<<<<<<<<<<<o                            o>>>>>>>>>>>
    #-----------o----------------------------o-----------
    #        endTime                    startTime
    
    if (thisDate == startDate) and (thisTime_last_block < startTime):
        return None
    
    if (thisDate == endDate) and (thisTime_first_block > endTime):
        return None
    
    if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
        return None
    
    return thisDatetime

def isFolderInDateRange(folder, startDate=None, endDate=None):
    """
    Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
    
    Inputs:
        folder            :    nombre completo del directorio.
                               Su formato deberia ser "/path_root/?YYYYDDD"
        
                                siendo:
                                    YYYY    :    Anio         (ejemplo 2015)
                                    DDD     :    Dia del anio (ejemplo 305)
        
        startDate          :    fecha inicial del rango seleccionado en formato datetime.date
        
        endDate            :    fecha final del rango seleccionado en formato datetime.date
    
    Return:
        Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                        fecha especificado, de lo contrario retorna False.
    Excepciones:
        Si el directorio no tiene el formato adecuado
    """
    
    basename = os.path.basename(folder)
    
    if not isRadarFolder(basename):
        print "The folder %s has not the rigth format" %folder
        return 0
    
    if startDate and endDate:
        thisDate = getDateFromRadarFolder(basename)
        
        if thisDate < startDate:
            return 0
        
        if thisDate > endDate:
            return 0
    
    return 1

def isFileInDateRange(filename, startDate=None, endDate=None):
    """
    Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
    
    Inputs:
        filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
        
                               Su formato deberia ser "?YYYYDDDsss"
        
                                siendo:
                                    YYYY    :    Anio         (ejemplo 2015)
                                    DDD     :    Dia del anio (ejemplo 305)
                                    sss     :    set
        
        startDate          :    fecha inicial del rango seleccionado en formato datetime.date
        
        endDate            :    fecha final del rango seleccionado en formato datetime.date
    
    Return:
        Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                        fecha especificado, de lo contrario retorna False.
    Excepciones:
        Si el archivo no tiene el formato adecuado
    """
    
    basename = os.path.basename(filename)
    
    if not isRadarFile(basename):
        print "The filename %s has not the rigth format" %filename
        return 0
    
    if startDate and endDate:
        thisDate = getDateFromRadarFile(basename)
        
        if thisDate < startDate:
            return 0
        
        if thisDate > endDate:
            return 0
    
    return 1

def getFileFromSet(path, ext, set):
    validFilelist = []
    fileList = os.listdir(path)
    
    # 0 1234 567 89A BCDE
    # H YYYY DDD SSS .ext
    
    for thisFile in fileList:
        try:
            year = int(thisFile[1:5])
            doy  = int(thisFile[5:8])
        except:
            continue
        
        if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
            continue
        
        validFilelist.append(thisFile)

    myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
    
    if len(myfile)!= 0:
        return myfile[0]
    else:
        filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
        print 'the filename %s does not exist'%filename
        print '...going to the last file: '
        
    if validFilelist:
        validFilelist = sorted( validFilelist, key=str.lower )
        return validFilelist[-1]

    return None

def getlastFileFromPath(path, ext):
    """
    Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
    al final de la depuracion devuelve el ultimo file de la lista que quedo.  
    
    Input: 
        fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
        ext         :    extension de los files contenidos en una carpeta 
            
    Return:
        El ultimo file de una determinada carpeta, no se considera el path.
    """
    validFilelist = []
    fileList = os.listdir(path)
    
    # 0 1234 567 89A BCDE
    # H YYYY DDD SSS .ext
    
    for thisFile in fileList:
        
        year = thisFile[1:5]
        if not isNumber(year):
            continue
        
        doy = thisFile[5:8]
        if not isNumber(doy):
            continue
        
        year = int(year)
        doy = int(doy)
        
        if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
            continue
        
        validFilelist.append(thisFile)

    if validFilelist:
        validFilelist = sorted( validFilelist, key=str.lower )
        return validFilelist[-1]

    return None

def checkForRealPath(path, foldercounter, year, doy, set, ext):
    """
    Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
    Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
    el path exacto de un determinado file.
    
    Example    :
        nombre correcto del file es  .../.../D2009307/P2009307367.ext
        
        Entonces la funcion prueba con las siguientes combinaciones
            .../.../y2009307367.ext
            .../.../Y2009307367.ext
            .../.../x2009307/y2009307367.ext
            .../.../x2009307/Y2009307367.ext
            .../.../X2009307/y2009307367.ext
            .../.../X2009307/Y2009307367.ext
        siendo para este caso, la ultima combinacion de letras, identica al file buscado 
        
    Return:
        Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file 
        caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas 
        para el filename  
    """
    fullfilename = None
    find_flag = False
    filename = None
    
    prefixDirList = [None,'d','D']
    if ext.lower() == ".r": #voltage
        prefixFileList = ['d','D']
    elif ext.lower() == ".pdata": #spectra
        prefixFileList = ['p','P']
    else:
        return None, filename
    
    #barrido por las combinaciones posibles        
    for prefixDir in prefixDirList:
        thispath = path
        if prefixDir != None:
            #formo el nombre del directorio xYYYYDDD (x=d o x=D)
            if foldercounter == 0:
                thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy )) 
            else:
                thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
        for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
            filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
            fullfilename = os.path.join( thispath, filename ) #formo el path completo
            
            if os.path.exists( fullfilename ): #verifico que exista
                find_flag = True
                break
        if find_flag:
            break

    if not(find_flag):
        return None, filename

    return fullfilename, filename

def isRadarFolder(folder):
    try:
        year = int(folder[1:5])
        doy = int(folder[5:8])
    except:
        return 0
    
    return 1

def isRadarFile(file):
        try:
            year = int(file[1:5])
            doy = int(file[5:8])
            set = int(file[8:11])
        except:
            return 0
    
        return 1

def getDateFromRadarFile(file):
    try:
        year = int(file[1:5])
        doy = int(file[5:8])
        set = int(file[8:11])
    except:
        return None

    thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
    return thisDate

def getDateFromRadarFolder(folder):
    try:
        year = int(folder[1:5])
        doy = int(folder[5:8])
    except:
        return None

    thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
    return thisDate
       
class JRODataIO:
    
    c = 3E8
    
    isConfig = False
    
    basicHeaderObj = None
    
    systemHeaderObj = None
    
    radarControllerHeaderObj = None
    
    processingHeaderObj = None
    
    dtype = None
    
    pathList = []
    
    filenameList = []
    
    filename = None
    
    ext = None
    
    flagIsNewFile = 1
    
    flagDiscontinuousBlock = 0
    
    flagIsNewBlock = 0
    
    fp = None
    
    firstHeaderSize = 0
    
    basicHeaderSize = 24
    
    versionFile = 1103
    
    fileSize = None
    
#     ippSeconds = None
    
    fileSizeByHeader = None
    
    fileIndex = None
    
    profileIndex = None
    
    blockIndex = None
    
    nTotalBlocks = None
    
    maxTimeStep = 30
    
    lastUTTime = None
    
    datablock = None
    
    dataOut = None
    
    blocksize = None
    
    getByBlock = False
    
    def __init__(self):
        
        raise NotImplementedError
    
    def run(self):
        
        raise NotImplementedError

    def getDtypeWidth(self):
        
        dtype_index = get_dtype_index(self.dtype)
        dtype_width = get_dtype_width(dtype_index)
        
        return dtype_width
    
class JRODataReader(JRODataIO):
    
    
    online = 0
    
    realtime = 0
    
    nReadBlocks = 0
    
    delay  = 10   #number of seconds waiting a new file
        
    nTries  = 3  #quantity tries
        
    nFiles = 3   #number of files for searching
    
    path = None
    
    foldercounter = 0
    
    flagNoMoreFiles = 0
    
    datetimeList = []
    
    __isFirstTimeOnline = 1
    
    __printInfo = True
    
    profileIndex = None
    
    nTxs = 1
    
    txIndex = None
    
    def __init__(self):
        
        """
        This class is used to find data files
        
        Example:
            reader = JRODataReader()
            fileList = reader.findDataFiles()
            
        """
        pass
    

    def createObjByDefault(self):
        """

        """
        raise NotImplementedError

    def getBlockDimension(self):
        
        raise NotImplementedError

    def __searchFilesOffLine(self,
                            path,
                            startDate=None,
                            endDate=None,
                            startTime=datetime.time(0,0,0),
                            endTime=datetime.time(23,59,59),
                            set=None,
                            expLabel='',
                            ext='.r',
                            walk=True):
        
        self.filenameList = []
        self.datetimeList = []
        
        pathList = []
        
        dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
        
        if dateList == []:
#             print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
            return None, None
        
        if len(dateList) > 1:
            print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
        else:
            print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
             
        filenameList = []
        datetimeList = []
        
        for thisPath in pathList:
#             thisPath = pathList[pathDict[file]]
            
            fileList = glob.glob1(thisPath, "*%s" %ext)
            fileList.sort()
        
            for file in fileList:
                
                filename = os.path.join(thisPath,file)
                
                if not isFileInDateRange(filename, startDate, endDate):
                    continue
                
                thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
                
                if not(thisDatetime):
                    continue
                
                filenameList.append(filename)
                datetimeList.append(thisDatetime)
                
        if not(filenameList):
            print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
            return None, None
        
        print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
        print
        
        for i in range(len(filenameList)):
            print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())

        self.filenameList = filenameList
        self.datetimeList = datetimeList
        
        return pathList, filenameList

    def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
        
        """
        Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
        devuelve el archivo encontrado ademas de otros datos.
        
        Input: 
            path           :    carpeta donde estan contenidos los files que contiene data
            
            expLabel        :     Nombre del subexperimento (subfolder)
            
            ext              :    extension de los files
            
            walk        :    Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)

        Return:
            directory   :    eL directorio donde esta el file encontrado
            filename    :    el ultimo file de una determinada carpeta
            year        :    el anho
            doy         :    el numero de dia del anho
            set         :    el set del archivo
            
            
        """
        if not os.path.isdir(path):
            return None, None, None, None, None, None
        
        dirList = []
            
        if not walk:
            fullpath = path
            foldercounter = 0
        else:
            #Filtra solo los directorios
            for thisPath in os.listdir(path):
                if not os.path.isdir(os.path.join(path,thisPath)):
                    continue
                if not isRadarFolder(thisPath):
                    continue
                
                dirList.append(thisPath)
            
            if not(dirList):
                return None, None, None, None, None, None
        
            dirList = sorted( dirList, key=str.lower )
                
            doypath = dirList[-1]
            foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
            fullpath = os.path.join(path, doypath, expLabel)
            
            
        print "[Reading] %s folder was found: " %(fullpath )

        if set == None:
            filename = getlastFileFromPath(fullpath, ext)
        else:
            filename = getFileFromSet(fullpath, ext, set)

        if not(filename):
            return None, None, None, None, None, None
        
        print "[Reading] %s file was found" %(filename)
        
        if not(self.__verifyFile(os.path.join(fullpath, filename))):
            return None, None, None, None, None, None

        year = int( filename[1:5] )
        doy  = int( filename[5:8] )
        set  = int( filename[8:11] )        
        
        return fullpath, foldercounter, filename, year, doy, set
    
    def __setNextFileOffline(self):
        
        idFile = self.fileIndex

        while (True):
            idFile += 1
            if not(idFile < len(self.filenameList)):
                self.flagNoMoreFiles = 1
#                 print "[Reading] No more Files"
                return 0

            filename = self.filenameList[idFile]

            if not(self.__verifyFile(filename)):
                continue

            fileSize = os.path.getsize(filename)
            fp = open(filename,'rb')
            break

        self.flagIsNewFile = 1
        self.fileIndex = idFile
        self.filename = filename
        self.fileSize = fileSize
        self.fp = fp

#         print "[Reading] Setting the file: %s"%self.filename

        return 1

    def __setNextFileOnline(self):
        """
        Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
        no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
        siguientes.   
            
        Affected: 
            self.flagIsNewFile
            self.filename
            self.fileSize
            self.fp
            self.set
            self.flagNoMoreFiles

        Return: 
            0    : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
            1    : si el file fue abierto con exito y esta listo a ser leido
        
        Excepciones: 
            Si un determinado file no puede ser abierto
        """
        nFiles = 0
        fileOk_flag = False        
        firstTime_flag = True

        self.set += 1
        
        if self.set > 999:
            self.set = 0
            self.foldercounter += 1 
        
        #busca el 1er file disponible
        fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
        if fullfilename:
            if self.__verifyFile(fullfilename, False):
                fileOk_flag = True

        #si no encuentra un file entonces espera y vuelve a buscar
        if not(fileOk_flag): 
            for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles

                if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces  
                    tries = self.nTries
                else:
                    tries = 1 #si no es la 1era vez entonces solo lo hace una vez
                    
                for nTries in range( tries ): 
                    if firstTime_flag:
                        print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 ) 
                        sleep( self.delay )
                    else:
                        print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
                    
                    fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                    if fullfilename:
                        if self.__verifyFile(fullfilename):
                            fileOk_flag = True
                            break
                    
                if fileOk_flag:
                    break

                firstTime_flag = False

                print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
                self.set += 1
                    
                if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
                    self.set = 0
                    self.doy += 1
                    self.foldercounter = 0

        if fileOk_flag:
            self.fileSize = os.path.getsize( fullfilename )
            self.filename = fullfilename
            self.flagIsNewFile = 1
            if self.fp != None: self.fp.close() 
            self.fp = open(fullfilename, 'rb')
            self.flagNoMoreFiles = 0
#             print '[Reading] Setting the file: %s' % fullfilename
        else:
            self.fileSize = 0
            self.filename = None
            self.flagIsNewFile = 0
            self.fp = None
            self.flagNoMoreFiles = 1
#             print '[Reading] No more files to read'

        return fileOk_flag
    
    def setNextFile(self):
        if self.fp != None:
            self.fp.close()

        if self.online:
            newFile = self.__setNextFileOnline()
        else:
            newFile = self.__setNextFileOffline()

        if not(newFile):
            print '[Reading] No more files to read'
            return 0

        print '[Reading] Setting the file: %s' % self.filename
        
        self.__readFirstHeader()
        self.nReadBlocks = 0
        return 1

    def __waitNewBlock(self):
        """
        Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma. 
        
        Si el modo de lectura es OffLine siempre retorn 0
        """
        if not self.online:
            return 0
        
        if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
            return 0
        
        currentPointer = self.fp.tell()
        
        neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
        
        for nTries in range( self.nTries ):
            
            self.fp.close()
            self.fp = open( self.filename, 'rb' )
            self.fp.seek( currentPointer )

            self.fileSize = os.path.getsize( self.filename )
            currentSize = self.fileSize - currentPointer

            if ( currentSize >= neededSize ):
                self.basicHeaderObj.read(self.fp)
                return 1
            
            if self.fileSize == self.fileSizeByHeader:
#                self.flagEoF = True
                return 0
            
            print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
            sleep( self.delay )
            
        
        return 0        

    def waitDataBlock(self,pointer_location):
        
        currentPointer = pointer_location
        
        neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
        
        for nTries in range( self.nTries ):
            self.fp.close()
            self.fp = open( self.filename, 'rb' )
            self.fp.seek( currentPointer )
            
            self.fileSize = os.path.getsize( self.filename )
            currentSize = self.fileSize - currentPointer
            
            if ( currentSize >= neededSize ):
                return 1
            
            print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
            sleep( self.delay )
        
        return 0

    def __jumpToLastBlock(self):
        
        if not(self.__isFirstTimeOnline):
            return
        
        csize = self.fileSize - self.fp.tell()
        blocksize = self.processingHeaderObj.blockSize
        
        #salta el primer bloque de datos
        if csize > self.processingHeaderObj.blockSize:
            self.fp.seek(self.fp.tell() + blocksize)
        else:
            return
        
        csize = self.fileSize - self.fp.tell()
        neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
        while True:
            
            if self.fp.tell()<self.fileSize:
                self.fp.seek(self.fp.tell() + neededsize)
            else:
                self.fp.seek(self.fp.tell() - neededsize)
                break
        
#        csize = self.fileSize - self.fp.tell()
#        neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
#        factor = int(csize/neededsize)
#        if factor > 0:
#            self.fp.seek(self.fp.tell() + factor*neededsize)
        
        self.flagIsNewFile = 0
        self.__isFirstTimeOnline = 0

    def __setNewBlock(self):
        
        if self.fp == None:
            return 0
            
#         if self.online:
#             self.__jumpToLastBlock()
        
        if self.flagIsNewFile:
            self.lastUTTime = self.basicHeaderObj.utc
            return 1
        
        if self.realtime:
            self.flagDiscontinuousBlock = 1
            if not(self.setNextFile()):
                return 0
            else:
                return 1
            
        currentSize = self.fileSize - self.fp.tell()
        neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
        
        if (currentSize >= neededSize):
            self.basicHeaderObj.read(self.fp)
            self.lastUTTime = self.basicHeaderObj.utc
            return 1
        
        if self.__waitNewBlock():
            self.lastUTTime = self.basicHeaderObj.utc
            return 1

        if not(self.setNextFile()):
            return 0

        deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
        self.lastUTTime = self.basicHeaderObj.utc
        
        self.flagDiscontinuousBlock = 0

        if deltaTime > self.maxTimeStep:
            self.flagDiscontinuousBlock = 1

        return 1

    def readNextBlock(self):
        
        #Skip block out of startTime and endTime
        while True:
            if not(self.__setNewBlock()):
                return 0
    
            if not(self.readBlock()):
                return 0
            
            self.getBasicHeader()
        
            if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
                
                print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
                                                              self.processingHeaderObj.dataBlocksPerFile,
                                                              self.dataOut.datatime.ctime())
                continue
                
            break
        
        print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
                                                  self.processingHeaderObj.dataBlocksPerFile,
                                                  self.dataOut.datatime.ctime())
        return 1

    def __readFirstHeader(self):
        
        self.basicHeaderObj.read(self.fp)
        self.systemHeaderObj.read(self.fp)
        self.radarControllerHeaderObj.read(self.fp)
        self.processingHeaderObj.read(self.fp)

        self.firstHeaderSize = self.basicHeaderObj.size

        datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
        if datatype == 0:
            datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
        elif datatype == 1:
            datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
        elif datatype == 2:
            datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
        elif datatype == 3:
            datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
        elif datatype == 4:
            datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
        elif datatype == 5:
            datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
        else:
            raise ValueError, 'Data type was not defined'

        self.dtype = datatype_str
        #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
        self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
#        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
#        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
        self.getBlockDimension()
    
    def __verifyFile(self, filename, msgFlag=True):
        
        msg = None
        
        try:
            fp = open(filename, 'rb')
        except IOError:
            
            if msgFlag:
                print "[Reading] File %s can't be opened" % (filename)
                
            return False
        
        currentPosition = fp.tell()
        neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize

        if neededSize == 0:
            basicHeaderObj = BasicHeader(LOCALTIME)
            systemHeaderObj = SystemHeader()
            radarControllerHeaderObj = RadarControllerHeader()
            processingHeaderObj = ProcessingHeader()
            
            if not( basicHeaderObj.read(fp) ):
                fp.close()
                return False
            
            if not( systemHeaderObj.read(fp) ):
                fp.close()
                return False
            
            if not( radarControllerHeaderObj.read(fp) ):
                fp.close()
                return False
            
            if not( processingHeaderObj.read(fp) ):
                fp.close()
                return False
            
            neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
        else:
            msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename

        fp.close()
        
        fileSize = os.path.getsize(filename)
        currentSize = fileSize - currentPosition
        
        if currentSize < neededSize:
            if msgFlag and (msg != None):
                print msg
            return False

        return True

    def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
        
        path_empty = True
        
        dateList = []
        pathList = []
        
        multi_path = path.split(',')
        
        if not walk:
    
            for single_path in multi_path:
                
                if not os.path.isdir(single_path):
                    continue
                
                fileList = glob.glob1(single_path, "*"+ext)
                
                if not fileList:
                    continue
                
                path_empty = False
                
                fileList.sort()
                
                for thisFile in fileList:
                    
                    if not os.path.isfile(os.path.join(single_path, thisFile)):
                        continue
                    
                    if not isRadarFile(thisFile):
                        continue
                    
                    if not isFileInDateRange(thisFile, startDate, endDate):
                        continue
                    
                    thisDate = getDateFromRadarFile(thisFile)
                    
                    if thisDate in dateList:
                        continue
                    
                    dateList.append(thisDate)
                    pathList.append(single_path)
        
        else:
            for single_path in multi_path:
                
                if not os.path.isdir(single_path):
                    continue
                    
                dirList = []
                
                for thisPath in os.listdir(single_path):
                    
                    if not os.path.isdir(os.path.join(single_path,thisPath)):
                        continue
                    
                    if not isRadarFolder(thisPath):
                        continue
                    
                    if not isFolderInDateRange(thisPath, startDate, endDate):
                        continue
                    
                    dirList.append(thisPath)
        
                if not dirList:
                    continue
                
                dirList.sort()
                
                for thisDir in dirList:
                    
                    datapath = os.path.join(single_path, thisDir, expLabel)
                    fileList = glob.glob1(datapath, "*"+ext)
                    
                    if not fileList:
                        continue
                    
                    path_empty = False
                    
                    thisDate = getDateFromRadarFolder(thisDir)
                    
                    pathList.append(datapath)
                    dateList.append(thisDate)
        
        dateList.sort()
        
        if walk:
            pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
        else:
            pattern_path = multi_path[0]
        
        if path_empty:
            print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
        else:
            if not dateList:
                print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)

        if include_path:
            return dateList, pathList
        
        return dateList
        
    def setup(self,
                path=None,
                startDate=None, 
                endDate=None, 
                startTime=datetime.time(0,0,0), 
                endTime=datetime.time(23,59,59), 
                set=None, 
                expLabel = "", 
                ext = None, 
                online = False,
                delay = 60,
                walk = True,
                getblock = False,
                nTxs = 1,
                realtime=False):

        if path == None:
            raise ValueError, "[Reading] The path is not valid"

        if ext == None:
            ext = self.ext

        if online:
            print "[Reading] Searching files in online mode..."  
                   
            for nTries in range( self.nTries ):
                fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
                
                if fullpath:
                    break
                
                print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
                sleep( self.delay )
            
            if not(fullpath):
                print "[Reading] There 'isn't any valid file in %s" % path
                return
        
            self.year = year
            self.doy  = doy
            self.set  = set - 1
            self.path = path
            self.foldercounter = foldercounter
            last_set = None
        
        else:
            print "[Reading] Searching files in offline mode ..."
            pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                               startTime=startTime, endTime=endTime,
                                                               set=set, expLabel=expLabel, ext=ext,
                                                               walk=walk)
            
            if not(pathList):
#                 print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
#                                                         datetime.datetime.combine(startDate,startTime).ctime(),
#                                                         datetime.datetime.combine(endDate,endTime).ctime())
                 
#                 sys.exit(-1)
                
                self.fileIndex = -1
                self.pathList = []
                self.filenameList = []
                return
            
            self.fileIndex = -1
            self.pathList = pathList
            self.filenameList = filenameList
            file_name = os.path.basename(filenameList[-1])
            basename, ext = os.path.splitext(file_name)
            last_set = int(basename[-3:])
            
        self.online = online
        self.realtime = realtime
        self.delay = delay
        ext = ext.lower()
        self.ext = ext
        self.getByBlock = getblock
        self.nTxs = nTxs
        self.startTime = startTime
        self.endTime = endTime
        
        if not(self.setNextFile()):
            if (startDate!=None) and (endDate!=None):
                print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
            elif startDate != None:
                print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
            else:
                print "[Reading] No files"
                
                self.fileIndex = -1
                self.pathList = []
                self.filenameList = []
                return

#         self.getBasicHeader()
        
        if last_set != None:
            self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
        return

    def getBasicHeader(self):
        
        self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
        
        self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
        
        self.dataOut.timeZone = self.basicHeaderObj.timeZone
    
        self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
        
        self.dataOut.errorCount = self.basicHeaderObj.errorCount
        
        self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
        
        self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
        
        self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
        
        
    def getFirstHeader(self):
        
        raise NotImplementedError
    
    def getData(self):
        
        raise NotImplementedError

    def hasNotDataInBuffer(self):
        
        raise NotImplementedError

    def readBlock(self):
        
        raise NotImplementedError
    
    def isEndProcess(self):
        
        return self.flagNoMoreFiles
    
    def printReadBlocks(self):
        
        print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
    
    def printTotalBlocks(self):
        
        print "[Reading] Number of read blocks %04d" %self.nTotalBlocks

    def printNumberOfBlock(self):
        
        if self.flagIsNewBlock:
            print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
                                                      self.processingHeaderObj.dataBlocksPerFile,
                                                      self.dataOut.datatime.ctime())

    def printInfo(self):
        
        if self.__printInfo == False:
            return
        
        self.basicHeaderObj.printInfo()
        self.systemHeaderObj.printInfo()
        self.radarControllerHeaderObj.printInfo()
        self.processingHeaderObj.printInfo()
        
        self.__printInfo = False
        
    
    def run(self, **kwargs):
        
        if not(self.isConfig):
            
#            self.dataOut = dataOut
            self.setup(**kwargs)
            self.isConfig = True
            
        self.getData()

class JRODataWriter(JRODataIO):

    """ 
    Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
    de los datos siempre se realiza por bloques. 
    """
    
    blockIndex = 0
    
    path = None
        
    setFile = None
    
    profilesPerBlock = None
    
    blocksPerFile = None
    
    nWriteBlocks = 0
    
    fileDate = None
    
    def __init__(self, dataOut=None):
        raise NotImplementedError


    def hasAllDataInBuffer(self):
        raise NotImplementedError


    def setBlockDimension(self):
        raise NotImplementedError

    
    def writeBlock(self):
        raise NotImplementedError


    def putData(self):
        raise NotImplementedError

    
    def getProcessFlags(self):
        
        processFlags = 0
        
        dtype_index = get_dtype_index(self.dtype)
        procflag_dtype = get_procflag_dtype(dtype_index)
        
        processFlags += procflag_dtype
        
        if self.dataOut.flagDecodeData:
            processFlags += PROCFLAG.DECODE_DATA
        
        if self.dataOut.flagDeflipData:
            processFlags += PROCFLAG.DEFLIP_DATA
        
        if self.dataOut.code is not None:
            processFlags += PROCFLAG.DEFINE_PROCESS_CODE
        
        if self.dataOut.nCohInt > 1:
            processFlags += PROCFLAG.COHERENT_INTEGRATION
        
        if self.dataOut.type == "Spectra":
            if self.dataOut.nIncohInt > 1:
                processFlags += PROCFLAG.INCOHERENT_INTEGRATION
                
            if self.dataOut.data_dc is not None:
                processFlags += PROCFLAG.SAVE_CHANNELS_DC
            
            if self.dataOut.flagShiftFFT:
                processFlags += PROCFLAG.SHIFT_FFT_DATA
            
        return processFlags
    
    def setBasicHeader(self):
        
        self.basicHeaderObj.size = self.basicHeaderSize #bytes
        self.basicHeaderObj.version = self.versionFile
        self.basicHeaderObj.dataBlock = self.nTotalBlocks
        
        utc = numpy.floor(self.dataOut.utctime)
        milisecond  = (self.dataOut.utctime - utc)* 1000.0
        
        self.basicHeaderObj.utc = utc
        self.basicHeaderObj.miliSecond = milisecond
        self.basicHeaderObj.timeZone = self.dataOut.timeZone
        self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
        self.basicHeaderObj.errorCount = self.dataOut.errorCount
            
    def setFirstHeader(self):
        """
        Obtiene una copia del First Header
        
        Affected:
        
            self.basicHeaderObj
            self.systemHeaderObj
            self.radarControllerHeaderObj
            self.processingHeaderObj self.
            
        Return:
            None
        """
        
        raise NotImplementedError
           
    def __writeFirstHeader(self):
        """
        Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
        
        Affected:
            __dataType
            
        Return:
            None
        """
        
#        CALCULAR PARAMETROS
        
        sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
        self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
        
        self.basicHeaderObj.write(self.fp)
        self.systemHeaderObj.write(self.fp)
        self.radarControllerHeaderObj.write(self.fp)
        self.processingHeaderObj.write(self.fp)
        
    def __setNewBlock(self):
        """
        Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
        
        Return:
            0    :    si no pudo escribir nada
            1    :    Si escribio el Basic el First Header
        """        
        if self.fp == None:
            self.setNextFile()
        
        if self.flagIsNewFile:
            return 1
        
        if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
            self.basicHeaderObj.write(self.fp)
            return 1
        
        if not( self.setNextFile() ):
            return 0
        
        return 1


    def writeNextBlock(self):
        """
        Selecciona el bloque siguiente de datos y los escribe en un file
            
        Return: 
            0    :    Si no hizo pudo escribir el bloque de datos 
            1    :    Si no pudo escribir el bloque de datos
        """
        if not( self.__setNewBlock() ):
            return 0
        
        self.writeBlock()
        
        print "[Writing] Block No. %d/%d" %(self.blockIndex,
                                            self.processingHeaderObj.dataBlocksPerFile)
        
        return 1        

    def setNextFile(self):
        """ 
        Determina el siguiente file que sera escrito

        Affected: 
            self.filename
            self.subfolder
            self.fp
            self.setFile
            self.flagIsNewFile

        Return:
            0    :    Si el archivo no puede ser escrito
            1    :    Si el archivo esta listo para ser escrito
        """
        ext = self.ext
        path = self.path
        
        if self.fp != None:
            self.fp.close()
        
        timeTuple = time.localtime( self.dataOut.utctime)
        subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)

        fullpath = os.path.join( path, subfolder )
        setFile = self.setFile
        
        if not( os.path.exists(fullpath) ):
            os.mkdir(fullpath)
            setFile = -1 #inicializo mi contador de seteo
        else:
            filesList = os.listdir( fullpath )
            if len( filesList ) > 0:
                filesList = sorted( filesList, key=str.lower )
                filen = filesList[-1]
                # el filename debera tener el siguiente formato
                # 0 1234 567 89A BCDE (hex)
                # x YYYY DDD SSS .ext
                if isNumber( filen[8:11] ):
                    setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                else:    
                    setFile = -1
            else:
                setFile = -1 #inicializo mi contador de seteo
        
        setFile += 1
        
        #If this is a new day it resets some values
        if self.dataOut.datatime.date() > self.fileDate:
            setFile = 0
            self.nTotalBlocks = 0
                
        filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )

        filename = os.path.join( path, subfolder, filen )

        fp = open( filename,'wb' )
        
        self.blockIndex = 0
        
        #guardando atributos 
        self.filename = filename
        self.subfolder = subfolder
        self.fp = fp
        self.setFile = setFile
        self.flagIsNewFile = 1
        self.fileDate = self.dataOut.datatime.date()
        
        self.setFirstHeader()
        
        print '[Writing] Opening file: %s'%self.filename
        
        self.__writeFirstHeader()
        
        return 1
    
    def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
        """
        Setea el tipo de formato en la cual sera guardada la data y escribe el First Header 
            
        Inputs:
            path                :    directory where data will be saved
            profilesPerBlock    :    number of profiles per block 
            set                 :    initial file set
            datatype            :    An integer number that defines data type:
                                        0 : int8  (1 byte)
                                        1 : int16 (2 bytes)
                                        2 : int32 (4 bytes)
                                        3 : int64 (8 bytes)
                                        4 : float32 (4 bytes)
                                        5 : double64 (8 bytes)
            
        Return:
            0    :    Si no realizo un buen seteo
            1    :    Si realizo un buen seteo 
        """
        
        if ext == None:
            ext = self.ext
        
        self.ext = ext.lower()
        
        self.path = path
        
        if set is None:
            self.setFile = -1
        else:
            self.setFile = set - 1
        
        self.blocksPerFile = blocksPerFile
        
        self.profilesPerBlock = profilesPerBlock
        
        self.dataOut = dataOut
        self.fileDate = self.dataOut.datatime.date()
        #By default
        self.dtype = self.dataOut.dtype
        
        if datatype is not None:
            self.dtype = get_numpy_dtype(datatype)
        
        if not(self.setNextFile()):
            print "[Writing] There isn't a next file"
            return 0
        
        self.setBlockDimension()
        
        return 1
    
    def run(self, dataOut, **kwargs):
        
        if not(self.isConfig):
            
            self.setup(dataOut, **kwargs)
            self.isConfig = True
            
        self.putData()