''' $Author: murco $ $Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $ ''' import os, sys import glob import time import numpy import fnmatch import time, datetime import h5py import re from xml.etree.ElementTree import Element, SubElement, ElementTree try: import pyfits except: print "pyfits module has not been imported, it should be installed to save files in fits format" from jrodata import * from jroheaderIO import * from jroprocessing import * LOCALTIME = True #-18000 def isNumber(str): """ Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero. Excepciones: Si un determinado string no puede ser convertido a numero Input: str, string al cual se le analiza para determinar si convertible a un numero o no Return: True : si el string es uno numerico False : no es un string numerico """ try: float( str ) return True except: return False def isThisFileinRange(filename, startUTSeconds, endUTSeconds): """ Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado. Inputs: filename : nombre completo del archivo de datos en formato Jicamarca (.r) startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en segundos contados desde 01/01/1970. endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en segundos contados desde 01/01/1970. Return: Boolean : Retorna True si el archivo de datos contiene datos en el rango de fecha especificado, de lo contrario retorna False. Excepciones: Si el archivo no existe o no puede ser abierto Si la cabecera no puede ser leida. """ basicHeaderObj = BasicHeader(LOCALTIME) try: fp = open(filename,'rb') except: raise IOError, "The file %s can't be opened" %(filename) sts = basicHeaderObj.read(fp) fp.close() if not(sts): print "Skipping the file %s because it has not a valid header" %(filename) return 0 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)): return 0 return 1 def isFileinThisTime(filename, startTime, endTime): """ Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. Inputs: filename : nombre completo del archivo de datos en formato Jicamarca (.r) startTime : tiempo inicial del rango seleccionado en formato datetime.time endTime : tiempo final del rango seleccionado en formato datetime.time Return: Boolean : Retorna True si el archivo de datos contiene datos en el rango de fecha especificado, de lo contrario retorna False. Excepciones: Si el archivo no existe o no puede ser abierto Si la cabecera no puede ser leida. """ try: fp = open(filename,'rb') except: raise IOError, "The file %s can't be opened" %(filename) basicHeaderObj = BasicHeader(LOCALTIME) sts = basicHeaderObj.read(fp) fp.close() thisDatetime = basicHeaderObj.datatime thisTime = basicHeaderObj.datatime.time() if not(sts): print "Skipping the file %s because it has not a valid header" %(filename) return None if not ((startTime <= thisTime) and (endTime > thisTime)): return None return thisDatetime def getFileFromSet(path,ext,set): validFilelist = [] fileList = os.listdir(path) # 0 1234 567 89A BCDE # H YYYY DDD SSS .ext for file in fileList: try: year = int(file[1:5]) doy = int(file[5:8]) except: continue if (os.path.splitext(file)[-1].lower() != ext.lower()): continue validFilelist.append(file) myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set)) if len(myfile)!= 0: return myfile[0] else: filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower()) print 'the filename %s does not exist'%filename print '...going to the last file: ' if validFilelist: validFilelist = sorted( validFilelist, key=str.lower ) return validFilelist[-1] return None def getlastFileFromPath(path, ext): """ Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext" al final de la depuracion devuelve el ultimo file de la lista que quedo. Input: fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta ext : extension de los files contenidos en una carpeta Return: El ultimo file de una determinada carpeta, no se considera el path. """ validFilelist = [] fileList = os.listdir(path) # 0 1234 567 89A BCDE # H YYYY DDD SSS .ext for file in fileList: try: year = int(file[1:5]) doy = int(file[5:8]) except: continue if (os.path.splitext(file)[-1].lower() != ext.lower()): continue validFilelist.append(file) if validFilelist: validFilelist = sorted( validFilelist, key=str.lower ) return validFilelist[-1] return None def checkForRealPath(path, foldercounter, year, doy, set, ext): """ Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path, Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar el path exacto de un determinado file. Example : nombre correcto del file es .../.../D2009307/P2009307367.ext Entonces la funcion prueba con las siguientes combinaciones .../.../y2009307367.ext .../.../Y2009307367.ext .../.../x2009307/y2009307367.ext .../.../x2009307/Y2009307367.ext .../.../X2009307/y2009307367.ext .../.../X2009307/Y2009307367.ext siendo para este caso, la ultima combinacion de letras, identica al file buscado Return: Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas para el filename """ fullfilename = None find_flag = False filename = None prefixDirList = [None,'d','D'] if ext.lower() == ".r": #voltage prefixFileList = ['d','D'] elif ext.lower() == ".pdata": #spectra prefixFileList = ['p','P'] else: return None, filename #barrido por las combinaciones posibles for prefixDir in prefixDirList: thispath = path if prefixDir != None: #formo el nombre del directorio xYYYYDDD (x=d o x=D) if foldercounter == 0: thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy )) else: thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter)) for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D" filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext fullfilename = os.path.join( thispath, filename ) #formo el path completo if os.path.exists( fullfilename ): #verifico que exista find_flag = True break if find_flag: break if not(find_flag): return None, filename return fullfilename, filename def isDoyFolder(folder): try: year = int(folder[1:5]) except: return 0 try: doy = int(folder[5:8]) except: return 0 return 1 class JRODataIO: c = 3E8 isConfig = False basicHeaderObj = BasicHeader(LOCALTIME) systemHeaderObj = SystemHeader() radarControllerHeaderObj = RadarControllerHeader() processingHeaderObj = ProcessingHeader() online = 0 dtype = None pathList = [] filenameList = [] filename = None ext = None flagIsNewFile = 1 flagTimeBlock = 0 flagIsNewBlock = 0 fp = None firstHeaderSize = 0 basicHeaderSize = 24 versionFile = 1103 fileSize = None ippSeconds = None fileSizeByHeader = None fileIndex = None profileIndex = None blockIndex = None nTotalBlocks = None maxTimeStep = 30 lastUTTime = None datablock = None dataOut = None blocksize = None def __init__(self): raise ValueError, "Not implemented" def run(self): raise ValueError, "Not implemented" def getOutput(self): return self.dataOut class JRODataReader(JRODataIO, ProcessingUnit): nReadBlocks = 0 delay = 10 #number of seconds waiting a new file nTries = 3 #quantity tries nFiles = 3 #number of files for searching path = None foldercounter = 0 flagNoMoreFiles = 0 datetimeList = [] __isFirstTimeOnline = 1 __printInfo = True profileIndex = None def __init__(self): """ """ raise ValueError, "This method has not been implemented" def createObjByDefault(self): """ """ raise ValueError, "This method has not been implemented" def getBlockDimension(self): raise ValueError, "No implemented" def __searchFilesOffLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59), set=None, expLabel='', ext='.r', walk=True): pathList = [] if not walk: #pathList.append(path) multi_path = path.split(',') for single_path in multi_path: pathList.append(single_path) else: #dirList = [] multi_path = path.split(',') for single_path in multi_path: dirList = [] for thisPath in os.listdir(single_path): if not os.path.isdir(os.path.join(single_path,thisPath)): continue if not isDoyFolder(thisPath): continue dirList.append(thisPath) if not(dirList): return None, None thisDate = startDate while(thisDate <= endDate): year = thisDate.timetuple().tm_year doy = thisDate.timetuple().tm_yday matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*') if len(matchlist) == 0: thisDate += datetime.timedelta(1) continue for match in matchlist: pathList.append(os.path.join(single_path,match,expLabel)) thisDate += datetime.timedelta(1) if pathList == []: print "Any folder was found for the date range: %s-%s" %(startDate, endDate) return None, None print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate) filenameList = [] datetimeList = [] pathDict = {} filenameList_to_sort = [] for i in range(len(pathList)): thisPath = pathList[i] fileList = glob.glob1(thisPath, "*%s" %ext) fileList.sort() pathDict.setdefault(fileList[0]) pathDict[fileList[0]] = i filenameList_to_sort.append(fileList[0]) filenameList_to_sort.sort() for file in filenameList_to_sort: thisPath = pathList[pathDict[file]] fileList = glob.glob1(thisPath, "*%s" %ext) fileList.sort() for file in fileList: filename = os.path.join(thisPath,file) thisDatetime = isFileinThisTime(filename, startTime, endTime) if not(thisDatetime): continue filenameList.append(filename) datetimeList.append(thisDatetime) if not(filenameList): print "Any file was found for the time range %s - %s" %(startTime, endTime) return None, None print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime) print for i in range(len(filenameList)): print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) self.filenameList = filenameList self.datetimeList = datetimeList return pathList, filenameList def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None): """ Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y devuelve el archivo encontrado ademas de otros datos. Input: path : carpeta donde estan contenidos los files que contiene data expLabel : Nombre del subexperimento (subfolder) ext : extension de los files walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath) Return: directory : eL directorio donde esta el file encontrado filename : el ultimo file de una determinada carpeta year : el anho doy : el numero de dia del anho set : el set del archivo """ dirList = [] if not walk: fullpath = path foldercounter = 0 else: #Filtra solo los directorios for thisPath in os.listdir(path): if not os.path.isdir(os.path.join(path,thisPath)): continue if not isDoyFolder(thisPath): continue dirList.append(thisPath) if not(dirList): return None, None, None, None, None, None dirList = sorted( dirList, key=str.lower ) doypath = dirList[-1] foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0 fullpath = os.path.join(path, doypath, expLabel) print "%s folder was found: " %(fullpath ) if set == None: filename = getlastFileFromPath(fullpath, ext) else: filename = getFileFromSet(fullpath, ext, set) if not(filename): return None, None, None, None, None, None print "%s file was found" %(filename) if not(self.__verifyFile(os.path.join(fullpath, filename))): return None, None, None, None, None, None year = int( filename[1:5] ) doy = int( filename[5:8] ) set = int( filename[8:11] ) return fullpath, foldercounter, filename, year, doy, set def __setNextFileOffline(self): idFile = self.fileIndex while (True): idFile += 1 if not(idFile < len(self.filenameList)): self.flagNoMoreFiles = 1 print "No more Files" return 0 filename = self.filenameList[idFile] if not(self.__verifyFile(filename)): continue fileSize = os.path.getsize(filename) fp = open(filename,'rb') break self.flagIsNewFile = 1 self.fileIndex = idFile self.filename = filename self.fileSize = fileSize self.fp = fp print "Setting the file: %s"%self.filename return 1 def __setNextFileOnline(self): """ Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files siguientes. Affected: self.flagIsNewFile self.filename self.fileSize self.fp self.set self.flagNoMoreFiles Return: 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado 1 : si el file fue abierto con exito y esta listo a ser leido Excepciones: Si un determinado file no puede ser abierto """ nFiles = 0 fileOk_flag = False firstTime_flag = True self.set += 1 if self.set > 999: self.set = 0 self.foldercounter += 1 #busca el 1er file disponible fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext ) if fullfilename: if self.__verifyFile(fullfilename, False): fileOk_flag = True #si no encuentra un file entonces espera y vuelve a buscar if not(fileOk_flag): for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces tries = self.nTries else: tries = 1 #si no es la 1era vez entonces solo lo hace una vez for nTries in range( tries ): if firstTime_flag: print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 ) time.sleep( self.delay ) else: print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext) fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext ) if fullfilename: if self.__verifyFile(fullfilename): fileOk_flag = True break if fileOk_flag: break firstTime_flag = False print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename self.set += 1 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta self.set = 0 self.doy += 1 self.foldercounter = 0 if fileOk_flag: self.fileSize = os.path.getsize( fullfilename ) self.filename = fullfilename self.flagIsNewFile = 1 if self.fp != None: self.fp.close() self.fp = open(fullfilename, 'rb') self.flagNoMoreFiles = 0 print 'Setting the file: %s' % fullfilename else: self.fileSize = 0 self.filename = None self.flagIsNewFile = 0 self.fp = None self.flagNoMoreFiles = 1 print 'No more Files' return fileOk_flag def setNextFile(self): if self.fp != None: self.fp.close() if self.online: newFile = self.__setNextFileOnline() else: newFile = self.__setNextFileOffline() if not(newFile): return 0 self.__readFirstHeader() self.nReadBlocks = 0 return 1 def __waitNewBlock(self): """ Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma. Si el modo de lectura es OffLine siempre retorn 0 """ if not self.online: return 0 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile): return 0 currentPointer = self.fp.tell() neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize for nTries in range( self.nTries ): self.fp.close() self.fp = open( self.filename, 'rb' ) self.fp.seek( currentPointer ) self.fileSize = os.path.getsize( self.filename ) currentSize = self.fileSize - currentPointer if ( currentSize >= neededSize ): self.__rdBasicHeader() return 1 if self.fileSize == self.fileSizeByHeader: # self.flagEoF = True return 0 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1) time.sleep( self.delay ) return 0 def waitDataBlock(self,pointer_location): currentPointer = pointer_location neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize for nTries in range( self.nTries ): self.fp.close() self.fp = open( self.filename, 'rb' ) self.fp.seek( currentPointer ) self.fileSize = os.path.getsize( self.filename ) currentSize = self.fileSize - currentPointer if ( currentSize >= neededSize ): return 1 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1) time.sleep( self.delay ) return 0 def __jumpToLastBlock(self): if not(self.__isFirstTimeOnline): return csize = self.fileSize - self.fp.tell() blocksize = self.processingHeaderObj.blockSize #salta el primer bloque de datos if csize > self.processingHeaderObj.blockSize: self.fp.seek(self.fp.tell() + blocksize) else: return csize = self.fileSize - self.fp.tell() neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize while True: if self.fp.tell() 0: # self.fp.seek(self.fp.tell() + factor*neededsize) self.flagIsNewFile = 0 self.__isFirstTimeOnline = 0 def __setNewBlock(self): if self.fp == None: return 0 if self.online: self.__jumpToLastBlock() if self.flagIsNewFile: return 1 self.lastUTTime = self.basicHeaderObj.utc currentSize = self.fileSize - self.fp.tell() neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize if (currentSize >= neededSize): self.__rdBasicHeader() return 1 if self.__waitNewBlock(): return 1 if not(self.setNextFile()): return 0 deltaTime = self.basicHeaderObj.utc - self.lastUTTime # self.flagTimeBlock = 0 if deltaTime > self.maxTimeStep: self.flagTimeBlock = 1 return 1 def readNextBlock(self): if not(self.__setNewBlock()): return 0 if not(self.readBlock()): return 0 return 1 def __rdProcessingHeader(self, fp=None): if fp == None: fp = self.fp self.processingHeaderObj.read(fp) def __rdRadarControllerHeader(self, fp=None): if fp == None: fp = self.fp self.radarControllerHeaderObj.read(fp) def __rdSystemHeader(self, fp=None): if fp == None: fp = self.fp self.systemHeaderObj.read(fp) def __rdBasicHeader(self, fp=None): if fp == None: fp = self.fp self.basicHeaderObj.read(fp) def __readFirstHeader(self): self.__rdBasicHeader() self.__rdSystemHeader() self.__rdRadarControllerHeader() self.__rdProcessingHeader() self.firstHeaderSize = self.basicHeaderObj.size datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR)) if datatype == 0: datatype_str = numpy.dtype([('real',' %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime()) self.dataOut.blocknow = self.basicHeaderObj.dataBlock def printInfo(self): if self.__printInfo == False: return self.basicHeaderObj.printInfo() self.systemHeaderObj.printInfo() self.radarControllerHeaderObj.printInfo() self.processingHeaderObj.printInfo() self.__printInfo = False def run(self, **kwargs): if not(self.isConfig): # self.dataOut = dataOut self.setup(**kwargs) self.isConfig = True self.getData() class JRODataWriter(JRODataIO, Operation): """ Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura de los datos siempre se realiza por bloques. """ blockIndex = 0 path = None setFile = None profilesPerBlock = None blocksPerFile = None nWriteBlocks = 0 def __init__(self, dataOut=None): raise ValueError, "Not implemented" def hasAllDataInBuffer(self): raise ValueError, "Not implemented" def setBlockDimension(self): raise ValueError, "Not implemented" def writeBlock(self): raise ValueError, "No implemented" def putData(self): raise ValueError, "No implemented" def setBasicHeader(self): self.basicHeaderObj.size = self.basicHeaderSize #bytes self.basicHeaderObj.version = self.versionFile self.basicHeaderObj.dataBlock = self.nTotalBlocks utc = numpy.floor(self.dataOut.utctime) milisecond = (self.dataOut.utctime - utc)* 1000.0 self.basicHeaderObj.utc = utc self.basicHeaderObj.miliSecond = milisecond self.basicHeaderObj.timeZone = self.dataOut.timeZone self.basicHeaderObj.dstFlag = self.dataOut.dstFlag self.basicHeaderObj.errorCount = self.dataOut.errorCount def setFirstHeader(self): """ Obtiene una copia del First Header Affected: self.basicHeaderObj self.systemHeaderObj self.radarControllerHeaderObj self.processingHeaderObj self. Return: None """ raise ValueError, "No implemented" def __writeFirstHeader(self): """ Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader) Affected: __dataType Return: None """ # CALCULAR PARAMETROS sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader self.basicHeaderObj.write(self.fp) self.systemHeaderObj.write(self.fp) self.radarControllerHeaderObj.write(self.fp) self.processingHeaderObj.write(self.fp) self.dtype = self.dataOut.dtype def __setNewBlock(self): """ Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header Return: 0 : si no pudo escribir nada 1 : Si escribio el Basic el First Header """ if self.fp == None: self.setNextFile() if self.flagIsNewFile: return 1 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile: self.basicHeaderObj.write(self.fp) return 1 if not( self.setNextFile() ): return 0 return 1 def writeNextBlock(self): """ Selecciona el bloque siguiente de datos y los escribe en un file Return: 0 : Si no hizo pudo escribir el bloque de datos 1 : Si no pudo escribir el bloque de datos """ if not( self.__setNewBlock() ): return 0 self.writeBlock() return 1 def setNextFile(self): """ Determina el siguiente file que sera escrito Affected: self.filename self.subfolder self.fp self.setFile self.flagIsNewFile Return: 0 : Si el archivo no puede ser escrito 1 : Si el archivo esta listo para ser escrito """ ext = self.ext path = self.path if self.fp != None: self.fp.close() timeTuple = time.localtime( self.dataOut.utctime) subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) fullpath = os.path.join( path, subfolder ) if not( os.path.exists(fullpath) ): os.mkdir(fullpath) self.setFile = -1 #inicializo mi contador de seteo else: filesList = os.listdir( fullpath ) if len( filesList ) > 0: filesList = sorted( filesList, key=str.lower ) filen = filesList[-1] # el filename debera tener el siguiente formato # 0 1234 567 89A BCDE (hex) # x YYYY DDD SSS .ext if isNumber( filen[8:11] ): self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file else: self.setFile = -1 else: self.setFile = -1 #inicializo mi contador de seteo setFile = self.setFile setFile += 1 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext ) filename = os.path.join( path, subfolder, file ) fp = open( filename,'wb' ) self.blockIndex = 0 #guardando atributos self.filename = filename self.subfolder = subfolder self.fp = fp self.setFile = setFile self.flagIsNewFile = 1 self.setFirstHeader() print 'Writing the file: %s'%self.filename self.__writeFirstHeader() return 1 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None): """ Setea el tipo de formato en la cual sera guardada la data y escribe el First Header Inputs: path : el path destino en el cual se escribiran los files a crear format : formato en el cual sera salvado un file set : el setebo del file Return: 0 : Si no realizo un buen seteo 1 : Si realizo un buen seteo """ if ext == None: ext = self.ext ext = ext.lower() self.ext = ext self.path = path self.setFile = set - 1 self.blocksPerFile = blocksPerFile self.profilesPerBlock = profilesPerBlock self.dataOut = dataOut if not(self.setNextFile()): print "There isn't a next file" return 0 self.setBlockDimension() return 1 def run(self, dataOut, **kwargs): if not(self.isConfig): self.setup(dataOut, **kwargs) self.isConfig = True self.putData() class VoltageReader(JRODataReader): """ Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones: perfiles*alturas*canales) son almacenados en la variable "buffer". perfiles * alturas * canales Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader, RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de datos desde el "buffer" cada vez que se ejecute el metodo "getData". Example: dpath = "/home/myuser/data" startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0) endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0) readerObj = VoltageReader() readerObj.setup(dpath, startTime, endTime) while(True): #to get one profile profile = readerObj.getData() #print the profile print profile #If you want to see all datablock print readerObj.datablock if readerObj.flagNoMoreFiles: break """ ext = ".r" optchar = "D" dataOut = None def __init__(self): """ Inicializador de la clase VoltageReader para la lectura de datos de voltage. Input: dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para almacenar un perfil de datos cada vez que se haga un requerimiento (getData). El perfil sera obtenido a partir del buffer de datos, si el buffer esta vacio se hara un nuevo proceso de lectura de un bloque de datos. Si este parametro no es pasado se creara uno internamente. Variables afectadas: self.dataOut Return: None """ self.isConfig = False self.datablock = None self.utc = 0 self.ext = ".r" self.optchar = "D" self.basicHeaderObj = BasicHeader(LOCALTIME) self.systemHeaderObj = SystemHeader() self.radarControllerHeaderObj = RadarControllerHeader() self.processingHeaderObj = ProcessingHeader() self.online = 0 self.fp = None self.idFile = None self.dtype = None self.fileSizeByHeader = None self.filenameList = [] self.filename = None self.fileSize = None self.firstHeaderSize = 0 self.basicHeaderSize = 24 self.pathList = [] self.filenameList = [] self.lastUTTime = 0 self.maxTimeStep = 30 self.flagNoMoreFiles = 0 self.set = 0 self.path = None self.profileIndex = 2**32-1 self.delay = 3 #seconds self.nTries = 3 #quantity tries self.nFiles = 3 #number of files for searching self.nReadBlocks = 0 self.flagIsNewFile = 1 self.__isFirstTimeOnline = 1 self.ippSeconds = 0 self.flagTimeBlock = 0 self.flagIsNewBlock = 0 self.nTotalBlocks = 0 self.blocksize = 0 self.dataOut = self.createObjByDefault() def createObjByDefault(self): dataObj = Voltage() return dataObj def __hasNotDataInBuffer(self): if self.profileIndex >= self.processingHeaderObj.profilesPerBlock: return 1 return 0 def getBlockDimension(self): """ Obtiene la cantidad de puntos a leer por cada bloque de datos Affected: self.blocksize Return: None """ pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels self.blocksize = pts2read def readBlock(self): """ readBlock lee el bloque de datos desde la posicion actual del puntero del archivo (self.fp) y actualiza todos los parametros relacionados al bloque de datos (metadata + data). La data leida es almacenada en el buffer y el contador del buffer es seteado a 0 Inputs: None Return: None Affected: self.profileIndex self.datablock self.flagIsNewFile self.flagIsNewBlock self.nTotalBlocks Exceptions: Si un bloque leido no es un bloque valido """ current_pointer_location = self.fp.tell() junk = numpy.fromfile( self.fp, self.dtype, self.blocksize ) try: junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) ) except: #print "The read block (%3d) has not enough data" %self.nReadBlocks if self.waitDataBlock(pointer_location=current_pointer_location): junk = numpy.fromfile( self.fp, self.dtype, self.blocksize ) junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) ) # return 0 junk = numpy.transpose(junk, (2,0,1)) self.datablock = junk['real'] + junk['imag']*1j self.profileIndex = 0 self.flagIsNewFile = 0 self.flagIsNewBlock = 1 self.nTotalBlocks += 1 self.nReadBlocks += 1 return 1 def getFirstHeader(self): self.dataOut.dtype = self.dtype self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight) self.dataOut.channelList = range(self.systemHeaderObj.nChannels) self.dataOut.ippSeconds = self.ippSeconds self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt self.dataOut.nCohInt = self.processingHeaderObj.nCohInt self.dataOut.flagShiftFFT = False if self.radarControllerHeaderObj.code != None: self.dataOut.nCode = self.radarControllerHeaderObj.nCode self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud self.dataOut.code = self.radarControllerHeaderObj.code self.dataOut.systemHeaderObj = self.systemHeaderObj.copy() self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy() self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip self.dataOut.flagShiftFFT = False def getData(self): """ getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock" Ademas incrementa el contador del buffer en 1. Return: data : retorna un perfil de voltages (alturas * canales) copiados desde el buffer. Si no hay mas archivos a leer retorna None. Variables afectadas: self.dataOut self.profileIndex Affected: self.dataOut self.profileIndex self.flagTimeBlock self.flagIsNewBlock """ if self.flagNoMoreFiles: self.dataOut.flagNoData = True print 'Process finished' return 0 self.flagTimeBlock = 0 self.flagIsNewBlock = 0 if self.__hasNotDataInBuffer(): if not( self.readNextBlock() ): return 0 self.getFirstHeader() if self.datablock == None: self.dataOut.flagNoData = True return 0 self.dataOut.data = self.datablock[:,self.profileIndex,:] self.dataOut.flagNoData = False self.getBasicHeader() self.profileIndex += 1 self.dataOut.realtime = self.online return self.dataOut.data class VoltageWriter(JRODataWriter): """ Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura de los datos siempre se realiza por bloques. """ ext = ".r" optchar = "D" shapeBuffer = None def __init__(self): """ Inicializador de la clase VoltageWriter para la escritura de datos de espectros. Affected: self.dataOut Return: None """ self.nTotalBlocks = 0 self.profileIndex = 0 self.isConfig = False self.fp = None self.flagIsNewFile = 1 self.nTotalBlocks = 0 self.flagIsNewBlock = 0 self.setFile = None self.dtype = None self.path = None self.filename = None self.basicHeaderObj = BasicHeader(LOCALTIME) self.systemHeaderObj = SystemHeader() self.radarControllerHeaderObj = RadarControllerHeader() self.processingHeaderObj = ProcessingHeader() def hasAllDataInBuffer(self): if self.profileIndex >= self.processingHeaderObj.profilesPerBlock: return 1 return 0 def setBlockDimension(self): """ Obtiene las formas dimensionales del los subbloques de datos que componen un bloque Affected: self.shape_spc_Buffer self.shape_cspc_Buffer self.shape_dc_Buffer Return: None """ self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) self.datablock = numpy.zeros((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights), dtype=numpy.dtype('complex64')) def writeBlock(self): """ Escribe el buffer en el file designado Affected: self.profileIndex self.flagIsNewFile self.flagIsNewBlock self.nTotalBlocks self.blockIndex Return: None """ data = numpy.zeros( self.shapeBuffer, self.dtype ) junk = numpy.transpose(self.datablock, (1,2,0)) data['real'] = junk.real data['imag'] = junk.imag data = data.reshape( (-1) ) data.tofile( self.fp ) self.datablock.fill(0) self.profileIndex = 0 self.flagIsNewFile = 0 self.flagIsNewBlock = 1 self.blockIndex += 1 self.nTotalBlocks += 1 def putData(self): """ Setea un bloque de datos y luego los escribe en un file Affected: self.flagIsNewBlock self.profileIndex Return: 0 : Si no hay data o no hay mas files que puedan escribirse 1 : Si se escribio la data de un bloque en un file """ if self.dataOut.flagNoData: return 0 self.flagIsNewBlock = 0 if self.dataOut.flagTimeBlock: self.datablock.fill(0) self.profileIndex = 0 self.setNextFile() if self.profileIndex == 0: self.setBasicHeader() self.datablock[:,self.profileIndex,:] = self.dataOut.data self.profileIndex += 1 if self.hasAllDataInBuffer(): #if self.flagIsNewFile: self.writeNextBlock() # self.setFirstHeader() return 1 def __getProcessFlags(self): processFlags = 0 dtype0 = numpy.dtype([('real',' 1: processFlags += PROCFLAG.COHERENT_INTEGRATION return processFlags def __getBlockSize(self): ''' Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage ''' dtype0 = numpy.dtype([('real',' 1: processFlags += PROCFLAG.INCOHERENT_INTEGRATION if self.dataOut.data_dc != None: processFlags += PROCFLAG.SAVE_CHANNELS_DC return processFlags def __getBlockSize(self): ''' Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra ''' dtype0 = numpy.dtype([('real',' 0: channelList = [] for channel in range(self.dataOut.nChannels): channelList.append(channel) channelList.append(channel) pairsList = [] if self.dataOut.nPairs > 0: for pair in self.dataOut.pairsList: pairsList.append(pair[0]) pairsList.append(pair[1]) spectraComb = channelList + pairsList spectraComb = numpy.array(spectraComb,dtype="u1") self.processingHeaderObj.spectraComb = spectraComb sizeOfSpcComb = len(spectraComb) processingHeaderSize += sizeOfSpcComb # The processing header should not have information about code # if self.dataOut.code != None: # self.processingHeaderObj.code = self.dataOut.code # self.processingHeaderObj.nCode = self.dataOut.nCode # self.processingHeaderObj.nBaud = self.dataOut.nBaud # nCodeSize = 4 # bytes # nBaudSize = 4 # bytes # codeSize = 4 # bytes # sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud) # processingHeaderSize += sizeOfCode if self.processingHeaderObj.nWindows != 0: self.processingHeaderObj.firstHeight = self.dataOut.heightList[0] self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0] self.processingHeaderObj.nHeights = self.dataOut.nHeights self.processingHeaderObj.samplesWin = self.dataOut.nHeights sizeOfFirstHeight = 4 sizeOfdeltaHeight = 4 sizeOfnHeights = 4 sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows processingHeaderSize += sizeOfWindows self.processingHeaderObj.size = processingHeaderSize class SpectraHeisWriter(Operation): # set = None setFile = None idblock = None doypath = None subfolder = None def __init__(self): self.wrObj = FITS() # self.dataOut = dataOut self.nTotalBlocks=0 # self.set = None self.setFile = None self.idblock = 0 self.wrpath = None self.doypath = None self.subfolder = None self.isConfig = False def isNumber(str): """ Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero. Excepciones: Si un determinado string no puede ser convertido a numero Input: str, string al cual se le analiza para determinar si convertible a un numero o no Return: True : si el string es uno numerico False : no es un string numerico """ try: float( str ) return True except: return False def setup(self, dataOut, wrpath): if not(os.path.exists(wrpath)): os.mkdir(wrpath) self.wrpath = wrpath # self.setFile = 0 self.dataOut = dataOut def putData(self): name= time.localtime( self.dataOut.utctime) ext=".fits" if self.doypath == None: self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple())) self.doypath = os.path.join( self.wrpath, self.subfolder ) os.mkdir(self.doypath) if self.setFile == None: # self.set = self.dataOut.set self.setFile = 0 # if self.set != self.dataOut.set: ## self.set = self.dataOut.set # self.setFile = 0 #make the filename file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext) filename = os.path.join(self.wrpath,self.subfolder, file) idblock = numpy.array([self.idblock],dtype="int64") header=self.wrObj.cFImage(idblock=idblock, year=time.gmtime(self.dataOut.utctime).tm_year, month=time.gmtime(self.dataOut.utctime).tm_mon, day=time.gmtime(self.dataOut.utctime).tm_mday, hour=time.gmtime(self.dataOut.utctime).tm_hour, minute=time.gmtime(self.dataOut.utctime).tm_min, second=time.gmtime(self.dataOut.utctime).tm_sec) c=3E8 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0] freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000)) colList = [] colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq) colList.append(colFreq) nchannel=self.dataOut.nChannels for i in range(nchannel): col = self.wrObj.writeData(name="PCh"+str(i+1), format=str(self.dataOut.nFFTPoints)+'E', data=10*numpy.log10(self.dataOut.data_spc[i,:])) colList.append(col) data=self.wrObj.Ctable(colList=colList) self.wrObj.CFile(header,data) self.wrObj.wFile(filename) #update the setFile self.setFile += 1 self.idblock += 1 return 1 def run(self, dataOut, **kwargs): if not(self.isConfig): self.setup(dataOut, **kwargs) self.isConfig = True self.putData() class ParameterConf: ELEMENTNAME = 'Parameter' def __init__(self): self.name = '' self.value = '' def readXml(self, parmElement): self.name = parmElement.get('name') self.value = parmElement.get('value') def getElementName(self): return self.ELEMENTNAME class Metadata: def __init__(self, filename): self.parmConfObjList = [] self.readXml(filename) def readXml(self, filename): self.projectElement = None self.procUnitConfObjDict = {} self.projectElement = ElementTree().parse(filename) self.project = self.projectElement.tag parmElementList = self.projectElement.getiterator(ParameterConf().getElementName()) for parmElement in parmElementList: parmConfObj = ParameterConf() parmConfObj.readXml(parmElement) self.parmConfObjList.append(parmConfObj) class FitsWriter(Operation): def __init__(self): self.isConfig = False self.dataBlocksPerFile = None self.blockIndex = 0 self.flagIsNewFile = 1 self.fitsObj = None self.optchar = 'P' self.ext = '.fits' self.setFile = 0 def setFitsHeader(self, dataOut, metadatafile): header_data = pyfits.PrimaryHDU() metadata4fits = Metadata(metadatafile) for parameter in metadata4fits.parmConfObjList: parm_name = parameter.name parm_value = parameter.value # if parm_value == 'fromdatadatetime': # value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple()) # elif parm_value == 'fromdataheights': # value = dataOut.nHeights # elif parm_value == 'fromdatachannel': # value = dataOut.nChannels # elif parm_value == 'fromdatasamples': # value = dataOut.nFFTPoints # else: # value = parm_value header_data.header[parm_name] = parm_value header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple()) header_data.header['CHANNELLIST'] = str(dataOut.channelList) header_data.header['NCHANNELS'] = dataOut.nChannels #header_data.header['HEIGHTS'] = dataOut.heightList header_data.header['NHEIGHTS'] = dataOut.nHeights header_data.header['IPPSECONDS'] = dataOut.ippSeconds header_data.header['NCOHINT'] = dataOut.nCohInt header_data.header['NINCOHINT'] = dataOut.nIncohInt header_data.header['TIMEZONE'] = dataOut.timeZone header_data.header['NBLOCK'] = self.blockIndex header_data.writeto(self.filename) self.addExtension(dataOut.heightList,'HEIGHTLIST') def setup(self, dataOut, path, dataBlocksPerFile, metadatafile): self.path = path self.dataOut = dataOut self.metadatafile = metadatafile self.dataBlocksPerFile = dataBlocksPerFile def open(self): self.fitsObj = pyfits.open(self.filename, mode='update') def addExtension(self, data, tagname): self.open() extension = pyfits.ImageHDU(data=data, name=tagname) #extension.header['TAG'] = tagname self.fitsObj.append(extension) self.write() def addData(self, data): self.open() extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE']) extension.header['UTCTIME'] = self.dataOut.utctime self.fitsObj.append(extension) self.blockIndex += 1 self.fitsObj[0].header['NBLOCK'] = self.blockIndex self.write() def write(self): self.fitsObj.flush(verbose=True) self.fitsObj.close() def setNextFile(self): ext = self.ext path = self.path timeTuple = time.localtime( self.dataOut.utctime) subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) fullpath = os.path.join( path, subfolder ) if not( os.path.exists(fullpath) ): os.mkdir(fullpath) self.setFile = -1 #inicializo mi contador de seteo else: filesList = os.listdir( fullpath ) if len( filesList ) > 0: filesList = sorted( filesList, key=str.lower ) filen = filesList[-1] if isNumber( filen[8:11] ): self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file else: self.setFile = -1 else: self.setFile = -1 #inicializo mi contador de seteo setFile = self.setFile setFile += 1 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext ) filename = os.path.join( path, subfolder, file ) self.blockIndex = 0 self.filename = filename self.setFile = setFile self.flagIsNewFile = 1 print 'Writing the file: %s'%self.filename self.setFitsHeader(self.dataOut, self.metadatafile) return 1 def writeBlock(self): self.addData(self.dataOut.data_spc) self.flagIsNewFile = 0 def __setNewBlock(self): if self.flagIsNewFile: return 1 if self.blockIndex < self.dataBlocksPerFile: return 1 if not( self.setNextFile() ): return 0 return 1 def writeNextBlock(self): if not( self.__setNewBlock() ): return 0 self.writeBlock() return 1 def putData(self): if self.flagIsNewFile: self.setNextFile() self.writeNextBlock() def run(self, dataOut, **kwargs): if not(self.isConfig): self.setup(dataOut, **kwargs) self.isConfig = True self.putData() class FitsReader(ProcessingUnit): # __TIMEZONE = time.timezone expName = None datetimestr = None utc = None nChannels = None nSamples = None dataBlocksPerFile = None comments = None lastUTTime = None header_dict = None data = None data_header_dict = None def __init__(self): self.isConfig = False self.ext = '.fits' self.setFile = 0 self.flagNoMoreFiles = 0 self.flagIsNewFile = 1 self.flagTimeBlock = None self.fileIndex = None self.filename = None self.fileSize = None self.fitsObj = None self.timeZone = None self.nReadBlocks = 0 self.nTotalBlocks = 0 self.dataOut = self.createObjByDefault() self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup() self.blockIndex = 1 def createObjByDefault(self): dataObj = Fits() return dataObj def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False): try: fitsObj = pyfits.open(filename,'readonly') except: raise IOError, "The file %s can't be opened" %(filename) header = fitsObj[0].header struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S") utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS ltc = utc if useLocalTime: ltc -= time.timezone thisDatetime = datetime.datetime.utcfromtimestamp(ltc) thisTime = thisDatetime.time() if not ((startTime <= thisTime) and (endTime > thisTime)): return None return thisDatetime def __setNextFileOnline(self): raise ValueError, "No implemented" def __setNextFileOffline(self): idFile = self.fileIndex while (True): idFile += 1 if not(idFile < len(self.filenameList)): self.flagNoMoreFiles = 1 print "No more Files" return 0 filename = self.filenameList[idFile] # if not(self.__verifyFile(filename)): # continue fileSize = os.path.getsize(filename) fitsObj = pyfits.open(filename,'readonly') break self.flagIsNewFile = 1 self.fileIndex = idFile self.filename = filename self.fileSize = fileSize self.fitsObj = fitsObj self.blockIndex = 0 print "Setting the file: %s"%self.filename return 1 def readHeader(self): headerObj = self.fitsObj[0] self.header_dict = headerObj.header if 'EXPNAME' in headerObj.header.keys(): self.expName = headerObj.header['EXPNAME'] if 'DATATYPE' in headerObj.header.keys(): self.dataType = headerObj.header['DATATYPE'] self.datetimestr = headerObj.header['DATETIME'] channelList = headerObj.header['CHANNELLIST'] channelList = channelList.split('[') channelList = channelList[1].split(']') channelList = channelList[0].split(',') channelList = [int(ch) for ch in channelList] self.channelList = channelList self.nChannels = headerObj.header['NCHANNELS'] self.nHeights = headerObj.header['NHEIGHTS'] self.ippSeconds = headerObj.header['IPPSECONDS'] self.nCohInt = headerObj.header['NCOHINT'] self.nIncohInt = headerObj.header['NINCOHINT'] self.dataBlocksPerFile = headerObj.header['NBLOCK'] self.timeZone = headerObj.header['TIMEZONE'] self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt if 'COMMENT' in headerObj.header.keys(): self.comments = headerObj.header['COMMENT'] self.readHeightList() def readHeightList(self): self.blockIndex = self.blockIndex + 1 obj = self.fitsObj[self.blockIndex] self.heightList = obj.data self.blockIndex = self.blockIndex + 1 def readExtension(self): obj = self.fitsObj[self.blockIndex] self.heightList = obj.data self.blockIndex = self.blockIndex + 1 def setNextFile(self): if self.online: newFile = self.__setNextFileOnline() else: newFile = self.__setNextFileOffline() if not(newFile): return 0 self.readHeader() self.nReadBlocks = 0 # self.blockIndex = 1 return 1 def __searchFilesOffLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59), set=None, expLabel='', ext='.fits', walk=True): pathList = [] if not walk: pathList.append(path) else: dirList = [] for thisPath in os.listdir(path): if not os.path.isdir(os.path.join(path,thisPath)): continue if not isDoyFolder(thisPath): continue dirList.append(thisPath) if not(dirList): return None, None thisDate = startDate while(thisDate <= endDate): year = thisDate.timetuple().tm_year doy = thisDate.timetuple().tm_yday matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*') if len(matchlist) == 0: thisDate += datetime.timedelta(1) continue for match in matchlist: pathList.append(os.path.join(path,match,expLabel)) thisDate += datetime.timedelta(1) if pathList == []: print "Any folder was found for the date range: %s-%s" %(startDate, endDate) return None, None print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate) filenameList = [] datetimeList = [] for i in range(len(pathList)): thisPath = pathList[i] fileList = glob.glob1(thisPath, "*%s" %ext) fileList.sort() for file in fileList: filename = os.path.join(thisPath,file) thisDatetime = self.isFileinThisTime(filename, startTime, endTime) if not(thisDatetime): continue filenameList.append(filename) datetimeList.append(thisDatetime) if not(filenameList): print "Any file was found for the time range %s - %s" %(startTime, endTime) return None, None print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime) print for i in range(len(filenameList)): print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) self.filenameList = filenameList self.datetimeList = datetimeList return pathList, filenameList def setup(self, path=None, startDate=None, endDate=None, startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59), set=0, expLabel = "", ext = None, online = False, delay = 60, walk = True): if path == None: raise ValueError, "The path is not valid" if ext == None: ext = self.ext if not(online): print "Searching files in offline mode ..." pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate, startTime=startTime, endTime=endTime, set=set, expLabel=expLabel, ext=ext, walk=walk) if not(pathList): print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path, datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()) sys.exit(-1) self.fileIndex = -1 self.pathList = pathList self.filenameList = filenameList self.online = online self.delay = delay ext = ext.lower() self.ext = ext if not(self.setNextFile()): if (startDate!=None) and (endDate!=None): print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()) elif startDate != None: print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime()) else: print "No files" sys.exit(-1) def readBlock(self): dataObj = self.fitsObj[self.blockIndex] self.data = dataObj.data self.data_header_dict = dataObj.header self.utc = self.data_header_dict['UTCTIME'] self.flagIsNewFile = 0 self.blockIndex += 1 self.nTotalBlocks += 1 self.nReadBlocks += 1 return 1 def __jumpToLastBlock(self): raise ValueError, "No implemented" def __waitNewBlock(self): """ Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma. Si el modo de lectura es OffLine siempre retorn 0 """ if not self.online: return 0 if (self.nReadBlocks >= self.dataBlocksPerFile): return 0 currentPointer = self.fp.tell() neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize for nTries in range( self.nTries ): self.fp.close() self.fp = open( self.filename, 'rb' ) self.fp.seek( currentPointer ) self.fileSize = os.path.getsize( self.filename ) currentSize = self.fileSize - currentPointer if ( currentSize >= neededSize ): self.__rdBasicHeader() return 1 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1) time.sleep( self.delay ) return 0 def __setNewBlock(self): if self.online: self.__jumpToLastBlock() if self.flagIsNewFile: return 1 self.lastUTTime = self.utc if self.online: if self.__waitNewBlock(): return 1 if self.nReadBlocks < self.dataBlocksPerFile: return 1 if not(self.setNextFile()): return 0 deltaTime = self.utc - self.lastUTTime self.flagTimeBlock = 0 if deltaTime > self.maxTimeStep: self.flagTimeBlock = 1 return 1 def readNextBlock(self): if not(self.__setNewBlock()): return 0 if not(self.readBlock()): return 0 return 1 def getData(self): if self.flagNoMoreFiles: self.dataOut.flagNoData = True print 'Process finished' return 0 self.flagTimeBlock = 0 self.flagIsNewBlock = 0 if not(self.readNextBlock()): return 0 if self.data == None: self.dataOut.flagNoData = True return 0 self.dataOut.data = self.data self.dataOut.data_header = self.data_header_dict self.dataOut.utctime = self.utc self.dataOut.header = self.header_dict self.dataOut.expName = self.expName self.dataOut.nChannels = self.nChannels self.dataOut.timeZone = self.timeZone self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile self.dataOut.comments = self.comments self.dataOut.timeInterval = self.timeInterval self.dataOut.channelList = self.channelList self.dataOut.heightList = self.heightList self.dataOut.flagNoData = False return self.dataOut.data def run(self, **kwargs): if not(self.isConfig): self.setup(**kwargs) self.isConfig = True self.getData() class RadacHeader(): def __init__(self, fp): header = 'Raw11/Data/RadacHeader' self.beamCodeByPulse = fp.get(header+'/BeamCode') self.beamCode = fp.get('Raw11/Data/Beamcodes') self.code = fp.get(header+'/Code') self.frameCount = fp.get(header+'/FrameCount') self.modeGroup = fp.get(header+'/ModeGroup') self.nsamplesPulse = fp.get(header+'/NSamplesPulse') self.pulseCount = fp.get(header+'/PulseCount') self.radacTime = fp.get(header+'/RadacTime') self.timeCount = fp.get(header+'/TimeCount') self.timeStatus = fp.get(header+'/TimeStatus') self.nrecords = self.pulseCount.shape[0] #numero de bloques self.npulses = self.pulseCount.shape[1] #numero de perfiles self.nsamples = self.nsamplesPulse[0,0] #numero de alturas self.nbeams = self.beamCode.shape[1] #numero de beams def getIndexRangeToPulse(self, idrecord=0): indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0) startPulseCountId = indexToZero[0][0] endPulseCountId = startPulseCountId - 1 range1 = numpy.arange(startPulseCountId,self.npulses,1) range2 = numpy.arange(0,startPulseCountId,1) return range1, range2 class AMISRReader(ProcessingUnit): path = None startDate = None endDate = None startTime = None endTime = None walk = None isConfig = False def __init__(self): self.set = None self.subset = None self.extension_file = '.h5' self.dtc_str = 'dtc' self.dtc_id = 0 self.status = True self.isConfig = False self.dirnameList = [] self.filenameList = [] self.fileIndex = None self.flagNoMoreFiles = False self.flagIsNewFile = 0 self.filename = '' self.amisrFilePointer = None self.radacHeaderObj = None self.dataOut = self.__createObjByDefault() self.datablock = None self.rest_datablock = None self.range = None self.idrecord_count = 0 self.profileIndex = 0 self.idpulse_range1 = None self.idpulse_range2 = None self.beamCodeByFrame = None self.radacTimeByFrame = None #atributos originales tal y como esta en el archivo de datos self.beamCodesFromFile = None self.radacTimeFromFile = None self.rangeFromFile = None self.dataByFrame = None self.dataset = None self.beamCodeDict = {} self.beamRangeDict = {} #experiment cgf file self.npulsesint_fromfile = None self.recordsperfile_fromfile = None self.nbeamcodes_fromfile = None self.ngates_fromfile = None self.ippSeconds_fromfile = None self.frequency_h5file = None def __createObjByDefault(self): dataObj = AMISR() return dataObj def __setParameters(self,path,startDate,endDate,startTime,endTime,walk): self.path = path self.startDate = startDate self.endDate = endDate self.startTime = startTime self.endTime = endTime self.walk = walk def __checkPath(self): if os.path.exists(self.path): self.status = 1 else: self.status = 0 print 'Path:%s does not exists'%self.path return def __selDates(self, amisr_dirname_format): year = int(amisr_dirname_format[0:4]) month = int(amisr_dirname_format[4:6]) dom = int(amisr_dirname_format[6:8]) thisDate = datetime.date(year,month,dom) if (thisDate>=self.startDate and thisDate <= self.endDate): return amisr_dirname_format def __findDataForDates(self): if not(self.status): return None pat = '\d+.\d+' dirnameList = [re.search(pat,x).string for x in os.listdir(self.path)] dirnameList = [self.__selDates(x) for x in dirnameList] dirnameList = filter(lambda x:x!=None,dirnameList) if len(dirnameList)>0: self.status = 1 self.dirnameList = dirnameList self.dirnameList.sort() else: self.status = 0 return None def __getTimeFromData(self): pass def __filterByGlob1(self, dirName): filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file) filterDict = {} filterDict.setdefault(dirName) filterDict[dirName] = filter_files return filterDict def __getFilenameList(self, fileListInKeys, dirList): for value in fileListInKeys: dirName = value.keys()[0] for file in value[dirName]: filename = os.path.join(dirName, file) self.filenameList.append(filename) def __selectDataForTimes(self): #aun no esta implementado el filtro for tiempo if not(self.status): return None dirList = [os.path.join(self.path,x) for x in self.dirnameList] fileListInKeys = [self.__filterByGlob1(x) for x in dirList] self.__getFilenameList(fileListInKeys, dirList) if len(self.filenameList)>0: self.status = 1 self.filenameList.sort() else: self.status = 0 return None def __searchFilesOffline(self, path, startDate, endDate, startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59), walk=True): self.__setParameters(path, startDate, endDate, startTime, endTime, walk) self.__checkPath() self.__findDataForDates() self.__selectDataForTimes() for i in range(len(self.filenameList)): print "%s" %(self.filenameList[i]) return def __setNextFileOffline(self): idFile = self.fileIndex while (True): idFile += 1 if not(idFile < len(self.filenameList)): self.flagNoMoreFiles = 1 print "No more Files" return 0 filename = self.filenameList[idFile] amisrFilePointer = h5py.File(filename,'r') break self.flagIsNewFile = 1 self.fileIndex = idFile self.filename = filename self.amisrFilePointer = amisrFilePointer print "Setting the file: %s"%self.filename return 1 def __readHeader(self): self.radacHeaderObj = RadacHeader(self.amisrFilePointer) #update values from experiment cfg file self.radacHeaderObj.nrecords = self.recordsperfile_fromfile self.radacHeaderObj.nbeams = self.nbeamcodes_fromfile self.radacHeaderObj.npulses = self.npulsesint_fromfile self.radacHeaderObj.nsamples = self.ngates_fromfile #get tuning frequency frequency_h5file_dataset = self.amisrFilePointer.get('Rx'+'/TuningFrequency') self.frequency_h5file = frequency_h5file_dataset[0,0] self.flagIsNewFile = 1 def __getBeamCode(self): self.beamCodeDict = {} self.beamRangeDict = {} for i in range(len(self.radacHeaderObj.beamCode[0,:])): self.beamCodeDict.setdefault(i) self.beamRangeDict.setdefault(i) self.beamCodeDict[i] = self.radacHeaderObj.beamCode[0,i] just4record0 = self.radacHeaderObj.beamCodeByPulse[0,:] for i in range(len(self.beamCodeDict.values())): xx = numpy.where(just4record0==self.beamCodeDict.values()[i]) self.beamRangeDict[i] = xx[0] def __getExpParameters(self): if not(self.status): return None experimentCfgPath = os.path.join(self.path, self.dirnameList[0], 'Setup') expFinder = glob.glob1(experimentCfgPath,'*.exp') if len(expFinder)== 0: self.status = 0 return None experimentFilename = os.path.join(experimentCfgPath,expFinder[0]) f = open(experimentFilename) lines = f.readlines() f.close() parmsList = ['npulsesint*','recordsperfile*','nbeamcodes*','ngates*'] filterList = [fnmatch.filter(lines, x) for x in parmsList] values = [re.sub(r'\D',"",x[0]) for x in filterList] self.npulsesint_fromfile = int(values[0]) self.recordsperfile_fromfile = int(values[1]) self.nbeamcodes_fromfile = int(values[2]) self.ngates_fromfile = int(values[3]) tufileFinder = fnmatch.filter(lines, 'tufile=*') tufile = tufileFinder[0].split('=')[1].split('\n')[0] tufilename = os.path.join(experimentCfgPath,tufile) f = open(tufilename) lines = f.readlines() f.close() self.ippSeconds_fromfile = float(lines[1].split()[2])/1E6 self.status = 1 def __setNextFile(self): newFile = self.__setNextFileOffline() if not(newFile): return 0 self.__readHeader() self.__getBeamCode() self.readDataBlock() def setup(self,path=None, startDate=None, endDate=None, startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59), walk=True): #Busqueda de archivos offline self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk) if not(self.filenameList): print "There is no files into the folder: %s"%(path) sys.exit(-1) self.__getExpParameters() self.fileIndex = -1 self.__setNextFile() def readRanges(self): dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range') #self.rangeFromFile = dataset.value self.rangeFromFile = numpy.reshape(dataset.value,(-1)) return range def readRadacTime(self,idrecord, range1, range2): self.radacTimeFromFile = self.radacHeaderObj.radacTime.value radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses)) #radacTimeByFrame = dataset[idrecord - 1,range1] #radacTimeByFrame = dataset[idrecord,range2] return radacTimeByFrame def readBeamCode(self, idrecord, range1, range2): dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode') beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses)) self.beamCodesFromFile = dataset.value #beamcodeByFrame[range1] = dataset[idrecord - 1, range1] #beamcodeByFrame[range2] = dataset[idrecord, range2] beamcodeByFrame[range1] = dataset[idrecord, range1] beamcodeByFrame[range2] = dataset[idrecord, range2] return beamcodeByFrame def __setDataByFrame(self): ndata = 2 # porque es complejo dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata)) return dataByFrame def __readDataSet(self): dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data') return dataset def __setDataBlock(self,): real = self.dataByFrame[:,:,0] #asumo que 0 es real imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario datablock = real + imag*1j #armo el complejo return datablock def readSamples_version1(self,idrecord): #estas tres primeras lineas solo se deben ejecutar una vez if self.flagIsNewFile: self.idpulse_range1, self.idpulse_range2 = self.radacHeaderObj.getIndexRangeToPulse(0) self.dataByFrame = self.__setDataByFrame() self.beamCodeByFrame = self.readBeamCode(idrecord, self.idpulse_range1, self.idpulse_range2) self.radacTimeByFrame = self.readRadacTime(idrecord, self.idpulse_range1, self.idpulse_range2) #reading dataset self.dataset = self.__readDataSet() if idrecord == 0: if len(numpy.where(self.dataByFrame!=0.0)[0]) or len(numpy.where(self.dataByFrame!=0.0)[1]) or len(numpy.where(self.dataByFrame!=0.0)[2]): #falta agregar una condicion para datos discontinuos #por defecto une los datos del record anterior self.dataByFrame[self.idpulse_range2, :, :] = self.dataset[idrecord, self.idpulse_range2, :, :] #timepulse self.radacTimeByFrame[self.idpulse_range2] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range2] else: self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord, self.idpulse_range1, :, :] self.radacTimeByFrame[self.idpulse_range1] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range1] datablock = self.__setDataBlock() return datablock self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord - 1,self.idpulse_range1, :, :] self.dataByFrame[self.idpulse_range2, :, :] = self.dataset[idrecord, self.idpulse_range2, :, :] datablock = self.__setDataBlock() self.flagIsNewFile = 0 self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord, self.idpulse_range1, :, :] return datablock def readSamples(self,idrecord): if self.flagIsNewFile: self.dataByFrame = self.__setDataByFrame() self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :] #reading ranges self.readRanges() #reading dataset self.dataset = self.__readDataSet() self.flagIsNewFile = 0 self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :] self.dataByFrame = self.dataset[idrecord, :, :, :] datablock = self.__setDataBlock() return datablock def readDataBlock(self): #self.datablock = self.readSamples(self.idrecord_count) self.datablock = self.readSamples(self.idrecord_count) #print 'record:', self.idrecord_count self.idrecord_count += 1 self.profileIndex = 0 if self.idrecord_count >= self.radacHeaderObj.nrecords: self.idrecord_count = 0 self.flagIsNewFile = 1 def readNextBlock(self): self.readDataBlock() if self.flagIsNewFile: self.__setNextFile() pass def __hasNotDataInBuffer(self): #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record if self.profileIndex >= self.radacHeaderObj.npulses: return 1 return 0 def printUTC(self): print self.dataOut.utctime print '' def setObjProperties(self): self.dataOut.heightList = self.rangeFromFile/1000.0 #km self.dataOut.nProfiles = self.radacHeaderObj.npulses self.dataOut.ippSeconds = self.ippSeconds_fromfile self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt self.dataOut.frequency = self.frequency_h5file self.dataOut.nBaud = None self.dataOut.nCode = None self.dataOut.code = None self.dataOut.beamCodeDict = self.beamCodeDict self.dataOut.beamRangeDict = self.beamRangeDict def getData(self): if self.flagNoMoreFiles: self.dataOut.flagNoData = True print 'Process finished' return 0 if self.__hasNotDataInBuffer(): self.readNextBlock() # if not( self.readNextBlock() ): # return 0 # self.getFirstHeader() if self.datablock == None: # setear esta condicion cuando no hayan datos por leers self.dataOut.flagNoData = True return 0 self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1)) self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex] self.dataOut.flagNoData = False self.profileIndex += 1 return self.dataOut.data def run(self, **kwargs): if not(self.isConfig): self.setup(**kwargs) self.setObjProperties() self.isConfig = True self.getData()