""" Created on Jul 2, 2014 @author: roj-idl71 """ import os import sys import glob import time import numpy import fnmatch import inspect import time import datetime import zmq from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width from schainpy.utils import log import schainpy.admin LOCALTIME = True DT_DIRECTIVES = { '%Y': 4, '%y': 2, '%m': 2, '%d': 2, '%j': 3, '%H': 2, '%M': 2, '%S': 2, '%f': 6 } def isNumber(cad): """ Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero. Excepciones: Si un determinado string no puede ser convertido a numero Input: str, string al cual se le analiza para determinar si convertible a un numero o no Return: True : si el string es uno numerico False : no es un string numerico """ try: float(cad) return True except: return False def isFileInEpoch(filename, startUTSeconds, endUTSeconds): """ Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado. Inputs: filename : nombre completo del archivo de datos en formato Jicamarca (.r) startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en segundos contados desde 01/01/1970. endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en segundos contados desde 01/01/1970. Return: Boolean : Retorna True si el archivo de datos contiene datos en el rango de fecha especificado, de lo contrario retorna False. Excepciones: Si el archivo no existe o no puede ser abierto Si la cabecera no puede ser leida. """ basicHeaderObj = BasicHeader(LOCALTIME) try: fp = open(filename, 'rb') except IOError: print("The file %s can't be opened" % (filename)) return 0 sts = basicHeaderObj.read(fp) fp.close() if not(sts): print("Skipping the file %s because it has not a valid header" % (filename)) return 0 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)): return 0 return 1 def isTimeInRange(thisTime, startTime, endTime): if endTime >= startTime: if (thisTime < startTime) or (thisTime > endTime): return 0 return 1 else: if (thisTime < startTime) and (thisTime > endTime): return 0 return 1 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime): """ Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. Inputs: filename : nombre completo del archivo de datos en formato Jicamarca (.r) startDate : fecha inicial del rango seleccionado en formato datetime.date endDate : fecha final del rango seleccionado en formato datetime.date startTime : tiempo inicial del rango seleccionado en formato datetime.time endTime : tiempo final del rango seleccionado en formato datetime.time Return: Boolean : Retorna True si el archivo de datos contiene datos en el rango de fecha especificado, de lo contrario retorna False. Excepciones: Si el archivo no existe o no puede ser abierto Si la cabecera no puede ser leida. """ try: fp = open(filename, 'rb') except IOError: print("The file %s can't be opened" % (filename)) return None firstBasicHeaderObj = BasicHeader(LOCALTIME) systemHeaderObj = SystemHeader() radarControllerHeaderObj = RadarControllerHeader() processingHeaderObj = ProcessingHeader() lastBasicHeaderObj = BasicHeader(LOCALTIME) sts = firstBasicHeaderObj.read(fp) if not(sts): print("[Reading] Skipping the file %s because it has not a valid header" % (filename)) return None if not systemHeaderObj.read(fp): return None if not radarControllerHeaderObj.read(fp): return None if not processingHeaderObj.read(fp): return None filesize = os.path.getsize(filename) offset = processingHeaderObj.blockSize + 24 # header size if filesize <= offset: print("[Reading] %s: This file has not enough data" % filename) return None fp.seek(-offset, 2) sts = lastBasicHeaderObj.read(fp) fp.close() thisDatetime = lastBasicHeaderObj.datatime thisTime_last_block = thisDatetime.time() thisDatetime = firstBasicHeaderObj.datatime thisDate = thisDatetime.date() thisTime_first_block = thisDatetime.time() # General case # o>>>>>>>>>>>>>><<<<<<<<<<<<<= startTime: if (thisTime_last_block < startTime) or (thisTime_first_block > endTime): return None return thisDatetime # If endTime < startTime then endTime belongs to the next day #<<<<<<<<<<>>>>>>>>>> #-----------o----------------------------o----------- # endTime startTime if (thisDate == startDate) and (thisTime_last_block < startTime): return None if (thisDate == endDate) and (thisTime_first_block > endTime): return None if (thisTime_last_block < startTime) and (thisTime_first_block > endTime): return None return thisDatetime def isFolderInDateRange(folder, startDate=None, endDate=None): """ Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. Inputs: folder : nombre completo del directorio. Su formato deberia ser "/path_root/?YYYYDDD" siendo: YYYY : Anio (ejemplo 2015) DDD : Dia del anio (ejemplo 305) startDate : fecha inicial del rango seleccionado en formato datetime.date endDate : fecha final del rango seleccionado en formato datetime.date Return: Boolean : Retorna True si el archivo de datos contiene datos en el rango de fecha especificado, de lo contrario retorna False. Excepciones: Si el directorio no tiene el formato adecuado """ basename = os.path.basename(folder) if not isRadarFolder(basename): print("The folder %s has not the rigth format" % folder) return 0 if startDate and endDate: thisDate = getDateFromRadarFolder(basename) if thisDate < startDate: return 0 if thisDate > endDate: return 0 return 1 def isFileInDateRange(filename, startDate=None, endDate=None): """ Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. Inputs: filename : nombre completo del archivo de datos en formato Jicamarca (.r) Su formato deberia ser "?YYYYDDDsss" siendo: YYYY : Anio (ejemplo 2015) DDD : Dia del anio (ejemplo 305) sss : set startDate : fecha inicial del rango seleccionado en formato datetime.date endDate : fecha final del rango seleccionado en formato datetime.date Return: Boolean : Retorna True si el archivo de datos contiene datos en el rango de fecha especificado, de lo contrario retorna False. Excepciones: Si el archivo no tiene el formato adecuado """ basename = os.path.basename(filename) if not isRadarFile(basename): print("The filename %s has not the rigth format" % filename) return 0 if startDate and endDate: thisDate = getDateFromRadarFile(basename) if thisDate < startDate: return 0 if thisDate > endDate: return 0 return 1 def getFileFromSet(path, ext, set): validFilelist = [] fileList = os.listdir(path) # 0 1234 567 89A BCDE # H YYYY DDD SSS .ext for thisFile in fileList: try: year = int(thisFile[1:5]) doy = int(thisFile[5:8]) except: continue if (os.path.splitext(thisFile)[-1].lower() != ext.lower()): continue validFilelist.append(thisFile) myfile = fnmatch.filter( validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set)) if len(myfile) != 0: return myfile[0] else: filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower()) print('the filename %s does not exist' % filename) print('...going to the last file: ') if validFilelist: validFilelist = sorted(validFilelist, key=str.lower) return validFilelist[-1] return None def getlastFileFromPath(path, ext): """ Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext" al final de la depuracion devuelve el ultimo file de la lista que quedo. Input: fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta ext : extension de los files contenidos en una carpeta Return: El ultimo file de una determinada carpeta, no se considera el path. """ validFilelist = [] fileList = os.listdir(path) # 0 1234 567 89A BCDE # H YYYY DDD SSS .ext for thisFile in fileList: year = thisFile[1:5] if not isNumber(year): continue doy = thisFile[5:8] if not isNumber(doy): continue year = int(year) doy = int(doy) if (os.path.splitext(thisFile)[-1].lower() != ext.lower()): continue validFilelist.append(thisFile) if validFilelist: validFilelist = sorted(validFilelist, key=str.lower) return validFilelist[-1] return None def isRadarFolder(folder): try: year = int(folder[1:5]) doy = int(folder[5:8]) except: return 0 return 1 def isRadarFile(file): try: year = int(file[1:5]) doy = int(file[5:8]) set = int(file[8:11]) except: return 0 return 1 def getDateFromRadarFile(file): try: year = int(file[1:5]) doy = int(file[5:8]) set = int(file[8:11]) except: return None thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1) return thisDate def getDateFromRadarFolder(folder): try: year = int(folder[1:5]) doy = int(folder[5:8]) except: return None thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1) return thisDate def parse_format(s, fmt): for i in range(fmt.count('%')): x = fmt.index('%') d = DT_DIRECTIVES[fmt[x:x+2]] fmt = fmt.replace(fmt[x:x+2], s[x:x+d]) return fmt class Reader(object): c = 3E8 isConfig = False dtype = None pathList = [] filenameList = [] datetimeList = [] filename = None ext = None flagIsNewFile = 1 flagDiscontinuousBlock = 0 flagIsNewBlock = 0 flagNoMoreFiles = 0 fp = None firstHeaderSize = 0 basicHeaderSize = 24 versionFile = 1103 fileSize = None fileSizeByHeader = None fileIndex = -1 profileIndex = None blockIndex = 0 nTotalBlocks = 0 maxTimeStep = 30 lastUTTime = None datablock = None dataOut = None getByBlock = False path = None startDate = None endDate = None startTime = datetime.time(0, 0, 0) endTime = datetime.time(23, 59, 59) set = None expLabel = "" online = False delay = 60 nTries = 3 # quantity tries nFiles = 3 # number of files for searching walk = True getblock = False nTxs = 1 realtime = False blocksize = 0 blocktime = None warnings = True verbose = True server = None format = None oneDDict = None twoDDict = None independentParam = None filefmt = None folderfmt = None open_file = open open_mode = 'rb' def run(self): raise NotImplementedError def getAllowedArgs(self): if hasattr(self, '__attrs__'): return self.__attrs__ else: return inspect.getargspec(self.run).args def set_kwargs(self, **kwargs): for key, value in kwargs.items(): setattr(self, key, value) def find_folders(self, path, startDate, endDate, folderfmt, last=False): folders = [x for f in path.split(',') for x in os.listdir(f) if os.path.isdir(os.path.join(f, x))] folders.sort() if last: folders = [folders[-1]] for folder in folders: try: dt = datetime.datetime.strptime(parse_format(folder, folderfmt), folderfmt).date() if dt >= startDate and dt <= endDate: yield os.path.join(path, folder) else: log.log('Skiping folder {}'.format(folder), self.name) except Exception as e: log.log('Skiping folder {}'.format(folder), self.name) continue return def find_files(self, folders, ext, filefmt, startDate=None, endDate=None, expLabel='', last=False): for path in folders: files = glob.glob1(path, '*{}'.format(ext)) files.sort() if last: if files: fo = files[-1] try: dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date() yield os.path.join(path, expLabel, fo) except Exception as e: pass return else: return for fo in files: try: dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date() if dt >= startDate and dt <= endDate: yield os.path.join(path, expLabel, fo) else: log.log('Skiping file {}'.format(fo), self.name) except Exception as e: log.log('Skiping file {}'.format(fo), self.name) continue def searchFilesOffLine(self, path, startDate, endDate, expLabel, ext, walk, filefmt, folderfmt): """Search files in offline mode for the given arguments Return: Generator of files """ if walk: folders = self.find_folders( path, startDate, endDate, folderfmt) else: folders = path.split(',') return self.find_files( folders, ext, filefmt, startDate, endDate, expLabel) def searchFilesOnLine(self, path, startDate, endDate, expLabel, ext, walk, filefmt, folderfmt): """Search for the last file of the last folder Arguments: path : carpeta donde estan contenidos los files que contiene data expLabel : Nombre del subexperimento (subfolder) ext : extension de los files walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath) Return: generator with the full path of last filename """ if walk: folders = self.find_folders( path, startDate, endDate, folderfmt, last=True) else: folders = path.split(',') return self.find_files( folders, ext, filefmt, startDate, endDate, expLabel, last=True) def setNextFile(self): """Set the next file to be readed open it and parse de file header""" while True: if self.fp != None: self.fp.close() if self.online: newFile = self.setNextFileOnline() else: newFile = self.setNextFileOffline() if not(newFile): if self.online: raise schainpy.admin.SchainError('Time to wait for new files reach') else: if self.fileIndex == -1: raise schainpy.admin.SchainWarning('No files found in the given path') else: raise schainpy.admin.SchainWarning('No more files to read') if self.verifyFile(self.filename): break log.log('Opening file: %s' % self.filename, self.name) self.readFirstHeader() self.nReadBlocks = 0 def setNextFileOnline(self): """Check for the next file to be readed in online mode. Set: self.filename self.fp self.filesize Return: boolean """ nextFile = True nextDay = False for nFiles in range(self.nFiles+1): for nTries in range(self.nTries): fullfilename, filename = self.checkForRealPath(nextFile, nextDay) if fullfilename is not None: break log.warning( "Waiting %0.2f sec for the next file: \"%s\" , try %02d ..." % (self.delay, filename, nTries + 1), self.name) time.sleep(self.delay) nextFile = False continue if fullfilename is not None: break self.nTries = 1 nextFile = True if nFiles == (self.nFiles - 1): log.log('Trying with next day...', self.name) nextDay = True self.nTries = 3 if fullfilename: self.fileSize = os.path.getsize(fullfilename) self.filename = fullfilename self.flagIsNewFile = 1 if self.fp != None: self.fp.close() self.fp = self.open_file(fullfilename, self.open_mode) self.flagNoMoreFiles = 0 self.fileIndex += 1 return 1 else: return 0 def setNextFileOffline(self): """Open the next file to be readed in offline mode""" try: filename = next(self.filenameList) self.fileIndex +=1 except StopIteration: self.flagNoMoreFiles = 1 return 0 self.filename = filename self.fileSize = os.path.getsize(filename) self.fp = self.open_file(filename, self.open_mode) self.flagIsNewFile = 1 return 1 @staticmethod def isDateTimeInRange(dt, startDate, endDate, startTime, endTime): """Check if the given datetime is in range""" if startDate <= dt.date() <= endDate: if startTime <= dt.time() <= endTime: return True return False def verifyFile(self, filename): """Check for a valid file Arguments: filename -- full path filename Return: boolean """ return True def checkForRealPath(self, nextFile, nextDay): """Check if the next file to be readed exists""" raise NotImplementedError def readFirstHeader(self): """Parse the file header""" pass class JRODataReader(Reader): utc = 0 nReadBlocks = 0 foldercounter = 0 firstHeaderSize = 0 basicHeaderSize = 24 __isFirstTimeOnline = 1 __printInfo = True filefmt = "*%Y%j***" folderfmt = "*%Y%j" def getDtypeWidth(self): dtype_index = get_dtype_index(self.dtype) dtype_width = get_dtype_width(dtype_index) return dtype_width def checkForRealPath(self, nextFile, nextDay): """Check if the next file to be readed exists. Example : nombre correcto del file es .../.../D2009307/P2009307367.ext Entonces la funcion prueba con las siguientes combinaciones .../.../y2009307367.ext .../.../Y2009307367.ext .../.../x2009307/y2009307367.ext .../.../x2009307/Y2009307367.ext .../.../X2009307/y2009307367.ext .../.../X2009307/Y2009307367.ext siendo para este caso, la ultima combinacion de letras, identica al file buscado Return: str -- fullpath of the file """ if nextFile: self.set += 1 if nextDay: self.set = 0 self.doy += 1 foldercounter = 0 prefixDirList = [None, 'd', 'D'] if self.ext.lower() == ".r": # voltage prefixFileList = ['d', 'D'] elif self.ext.lower() == ".pdata": # spectra prefixFileList = ['p', 'P'] # barrido por las combinaciones posibles for prefixDir in prefixDirList: thispath = self.path if prefixDir != None: # formo el nombre del directorio xYYYYDDD (x=d o x=D) if foldercounter == 0: thispath = os.path.join(self.path, "%s%04d%03d" % (prefixDir, self.year, self.doy)) else: thispath = os.path.join(self.path, "%s%04d%03d_%02d" % ( prefixDir, self.year, self.doy, foldercounter)) for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D" # formo el nombre del file xYYYYDDDSSS.ext filename = "%s%04d%03d%03d%s" % (prefixFile, self.year, self.doy, self.set, self.ext) fullfilename = os.path.join( thispath, filename) if os.path.exists(fullfilename): return fullfilename, filename return None, filename def __waitNewBlock(self): """ Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma. Si el modo de lectura es OffLine siempre retorn 0 """ if not self.online: return 0 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile): return 0 currentPointer = self.fp.tell() neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize for nTries in range(self.nTries): self.fp.close() self.fp = open(self.filename, 'rb') self.fp.seek(currentPointer) self.fileSize = os.path.getsize(self.filename) currentSize = self.fileSize - currentPointer if (currentSize >= neededSize): self.basicHeaderObj.read(self.fp) return 1 if self.fileSize == self.fileSizeByHeader: # self.flagEoF = True return 0 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)) time.sleep(self.delay) return 0 def waitDataBlock(self, pointer_location, blocksize=None): currentPointer = pointer_location if blocksize is None: neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize else: neededSize = blocksize for nTries in range(self.nTries): self.fp.close() self.fp = open(self.filename, 'rb') self.fp.seek(currentPointer) self.fileSize = os.path.getsize(self.filename) currentSize = self.fileSize - currentPointer if (currentSize >= neededSize): return 1 log.warning( "Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1), self.name ) time.sleep(self.delay) return 0 def __setNewBlock(self): if self.fp == None: return 0 if self.flagIsNewFile: self.lastUTTime = self.basicHeaderObj.utc return 1 if self.realtime: self.flagDiscontinuousBlock = 1 if not(self.setNextFile()): return 0 else: return 1 currentSize = self.fileSize - self.fp.tell() neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize if (currentSize >= neededSize): self.basicHeaderObj.read(self.fp) self.lastUTTime = self.basicHeaderObj.utc return 1 if self.__waitNewBlock(): self.lastUTTime = self.basicHeaderObj.utc return 1 if not(self.setNextFile()): return 0 deltaTime = self.basicHeaderObj.utc - self.lastUTTime self.lastUTTime = self.basicHeaderObj.utc self.flagDiscontinuousBlock = 0 if deltaTime > self.maxTimeStep: self.flagDiscontinuousBlock = 1 return 1 def readNextBlock(self): while True: self.__setNewBlock() if not(self.readBlock()): return 0 self.getBasicHeader() if not self.isDateTimeInRange(self.dataOut.datatime, self.startDate, self.endDate, self.startTime, self.endTime): print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks, self.processingHeaderObj.dataBlocksPerFile, self.dataOut.datatime.ctime())) continue break if self.verbose: print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks, self.processingHeaderObj.dataBlocksPerFile, self.dataOut.datatime.ctime())) return 1 def readFirstHeader(self): self.basicHeaderObj.read(self.fp) self.systemHeaderObj.read(self.fp) self.radarControllerHeaderObj.read(self.fp) self.processingHeaderObj.read(self.fp) self.firstHeaderSize = self.basicHeaderObj.size datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR)) if datatype == 0: datatype_str = numpy.dtype([('real', ' %s" %(self.nReadBlocks, # self.processingHeaderObj.dataBlocksPerFile, # self.dataOut.datatime.ctime()) def printInfo(self): if self.__printInfo == False: return self.basicHeaderObj.printInfo() self.systemHeaderObj.printInfo() self.radarControllerHeaderObj.printInfo() self.processingHeaderObj.printInfo() self.__printInfo = False def run(self, **kwargs): """ Arguments: path : startDate : endDate : startTime : endTime : set : expLabel : ext : online : delay : walk : getblock : nTxs : realtime : blocksize : blocktime : skip : cursor : warnings : server : verbose : format : oneDDict : twoDDict : independentParam : """ if not(self.isConfig): self.setup(**kwargs) self.isConfig = True if self.server is None: self.getData() else: self.getFromServer() class JRODataWriter(Reader): """ Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura de los datos siempre se realiza por bloques. """ setFile = None profilesPerBlock = None blocksPerFile = None nWriteBlocks = 0 fileDate = None def __init__(self, dataOut=None): raise NotImplementedError def hasAllDataInBuffer(self): raise NotImplementedError def setBlockDimension(self): raise NotImplementedError def writeBlock(self): raise NotImplementedError def putData(self): raise NotImplementedError def getDtypeWidth(self): dtype_index = get_dtype_index(self.dtype) dtype_width = get_dtype_width(dtype_index) return dtype_width def getProcessFlags(self): processFlags = 0 dtype_index = get_dtype_index(self.dtype) procflag_dtype = get_procflag_dtype(dtype_index) processFlags += procflag_dtype if self.dataOut.flagDecodeData: processFlags += PROCFLAG.DECODE_DATA if self.dataOut.flagDeflipData: processFlags += PROCFLAG.DEFLIP_DATA if self.dataOut.code is not None: processFlags += PROCFLAG.DEFINE_PROCESS_CODE if self.dataOut.nCohInt > 1: processFlags += PROCFLAG.COHERENT_INTEGRATION if self.dataOut.type == "Spectra": if self.dataOut.nIncohInt > 1: processFlags += PROCFLAG.INCOHERENT_INTEGRATION if self.dataOut.data_dc is not None: processFlags += PROCFLAG.SAVE_CHANNELS_DC if self.dataOut.flagShiftFFT: processFlags += PROCFLAG.SHIFT_FFT_DATA return processFlags def setBasicHeader(self): self.basicHeaderObj.size = self.basicHeaderSize # bytes self.basicHeaderObj.version = self.versionFile self.basicHeaderObj.dataBlock = self.nTotalBlocks utc = numpy.floor(self.dataOut.utctime) milisecond = (self.dataOut.utctime - utc) * 1000.0 self.basicHeaderObj.utc = utc self.basicHeaderObj.miliSecond = milisecond self.basicHeaderObj.timeZone = self.dataOut.timeZone self.basicHeaderObj.dstFlag = self.dataOut.dstFlag self.basicHeaderObj.errorCount = self.dataOut.errorCount def setFirstHeader(self): """ Obtiene una copia del First Header Affected: self.basicHeaderObj self.systemHeaderObj self.radarControllerHeaderObj self.processingHeaderObj self. Return: None """ raise NotImplementedError def __writeFirstHeader(self): """ Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader) Affected: __dataType Return: None """ # CALCULAR PARAMETROS sizeLongHeader = self.systemHeaderObj.size + \ self.radarControllerHeaderObj.size + self.processingHeaderObj.size self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader self.basicHeaderObj.write(self.fp) self.systemHeaderObj.write(self.fp) self.radarControllerHeaderObj.write(self.fp) self.processingHeaderObj.write(self.fp) def __setNewBlock(self): """ Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header Return: 0 : si no pudo escribir nada 1 : Si escribio el Basic el First Header """ if self.fp == None: self.setNextFile() if self.flagIsNewFile: return 1 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile: self.basicHeaderObj.write(self.fp) return 1 if not(self.setNextFile()): return 0 return 1 def writeNextBlock(self): """ Selecciona el bloque siguiente de datos y los escribe en un file Return: 0 : Si no hizo pudo escribir el bloque de datos 1 : Si no pudo escribir el bloque de datos """ if not(self.__setNewBlock()): return 0 self.writeBlock() print("[Writing] Block No. %d/%d" % (self.blockIndex, self.processingHeaderObj.dataBlocksPerFile)) return 1 def setNextFile(self): """Determina el siguiente file que sera escrito Affected: self.filename self.subfolder self.fp self.setFile self.flagIsNewFile Return: 0 : Si el archivo no puede ser escrito 1 : Si el archivo esta listo para ser escrito """ ext = self.ext path = self.path if self.fp != None: self.fp.close() if not os.path.exists(path): os.mkdir(path) timeTuple = time.localtime(self.dataOut.utctime) subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday) fullpath = os.path.join(path, subfolder) setFile = self.setFile if not(os.path.exists(fullpath)): os.mkdir(fullpath) setFile = -1 # inicializo mi contador de seteo else: filesList = os.listdir(fullpath) if len(filesList) > 0: filesList = sorted(filesList, key=str.lower) filen = filesList[-1] # el filename debera tener el siguiente formato # 0 1234 567 89A BCDE (hex) # x YYYY DDD SSS .ext if isNumber(filen[8:11]): # inicializo mi contador de seteo al seteo del ultimo file setFile = int(filen[8:11]) else: setFile = -1 else: setFile = -1 # inicializo mi contador de seteo setFile += 1 # If this is a new day it resets some values if self.dataOut.datatime.date() > self.fileDate: setFile = 0 self.nTotalBlocks = 0 filen = '{}{:04d}{:03d}{:03d}{}'.format( self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext) filename = os.path.join(path, subfolder, filen) fp = open(filename, 'wb') self.blockIndex = 0 self.filename = filename self.subfolder = subfolder self.fp = fp self.setFile = setFile self.flagIsNewFile = 1 self.fileDate = self.dataOut.datatime.date() self.setFirstHeader() print('[Writing] Opening file: %s' % self.filename) self.__writeFirstHeader() return 1 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4): """ Setea el tipo de formato en la cual sera guardada la data y escribe el First Header Inputs: path : directory where data will be saved profilesPerBlock : number of profiles per block set : initial file set datatype : An integer number that defines data type: 0 : int8 (1 byte) 1 : int16 (2 bytes) 2 : int32 (4 bytes) 3 : int64 (8 bytes) 4 : float32 (4 bytes) 5 : double64 (8 bytes) Return: 0 : Si no realizo un buen seteo 1 : Si realizo un buen seteo """ if ext == None: ext = self.ext self.ext = ext.lower() self.path = path if set is None: self.setFile = -1 else: self.setFile = set - 1 self.blocksPerFile = blocksPerFile self.profilesPerBlock = profilesPerBlock self.dataOut = dataOut self.fileDate = self.dataOut.datatime.date() self.dtype = self.dataOut.dtype if datatype is not None: self.dtype = get_numpy_dtype(datatype) if not(self.setNextFile()): print("[Writing] There isn't a next file") return 0 self.setBlockDimension() return 1 def run(self, dataOut, path, blocksPerFile=100, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs): if not(self.isConfig): self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs) self.isConfig = True self.dataOut = dataOut self.putData() return self.dataOut