diff --git a/schainpy/model/io/jroIO_base.py b/schainpy/model/io/jroIO_base.py index 5a84195..976b81e 100644 --- a/schainpy/model/io/jroIO_base.py +++ b/schainpy/model/io/jroIO_base.py @@ -541,8 +541,8 @@ class JRODataIO: return inspect.getargspec(self.run).args class JRODataReader(JRODataIO): - - + + firstTime = True online = 0 realtime = 0 @@ -578,8 +578,8 @@ class JRODataReader(JRODataIO): selBlocksize = None selBlocktime = None - - + + onlineWithDate = False def __init__(self): """ @@ -616,7 +616,6 @@ class JRODataReader(JRODataIO): cursor=None, skip=None, walk=True): - self.filenameList = [] self.datetimeList = [] @@ -682,34 +681,35 @@ class JRODataReader(JRODataIO): print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) self.filenameList = filenameList - self.datetimeList = datetimeList - + self.datetimeList = datetimeList return pathList, filenameList - def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None): - + def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None): + """ - Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y - devuelve el archivo encontrado ademas de otros datos. - - Input: - path : carpeta donde estan contenidos los files que contiene data - - expLabel : Nombre del subexperimento (subfolder) - - ext : extension de los files - - walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath) - - Return: - directory : eL directorio donde esta el file encontrado - filename : el ultimo file de una determinada carpeta - year : el anho - doy : el numero de dia del anho - set : el set del archivo - - + Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y + devuelve el archivo encontrado ademas de otros datos. + + Input: + path : carpeta donde estan contenidos los files que contiene data + + expLabel : Nombre del subexperimento (subfolder) + + ext : extension de los files + + walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath) + + Return: + directory : eL directorio donde esta el file encontrado + filename : el ultimo file de una determinada carpeta + year : el anho + doy : el numero de dia del anho + set : el set del archivo + + """ + pathList = None + filenameList = None if not os.path.isdir(path): return None, None, None, None, None, None @@ -719,7 +719,7 @@ class JRODataReader(JRODataIO): fullpath = path foldercounter = 0 else: - #Filtra solo los directorios + # Filtra solo los directorios for thisPath in os.listdir(path): if not os.path.isdir(os.path.join(path,thisPath)): continue @@ -755,7 +755,7 @@ class JRODataReader(JRODataIO): year = int( filename[1:5] ) doy = int( filename[5:8] ) - set = int( filename[8:11] ) + set = int( filename[8:11] ) return fullpath, foldercounter, filename, year, doy, set @@ -767,7 +767,7 @@ class JRODataReader(JRODataIO): idFile += 1 if not(idFile < len(self.filenameList)): self.flagNoMoreFiles = 1 - # print "[Reading] No more Files" + # print "[Reading] No more Files" return 0 filename = self.filenameList[idFile] @@ -785,31 +785,32 @@ class JRODataReader(JRODataIO): self.fileSize = fileSize self.fp = fp - # print "[Reading] Setting the file: %s"%self.filename + #print "[Reading] Setting the file: %s"%self.filename return 1 def __setNextFileOnline(self): """ - Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si - no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files - siguientes. - - Affected: - self.flagIsNewFile - self.filename - self.fileSize - self.fp - self.set - self.flagNoMoreFiles - - Return: - 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado - 1 : si el file fue abierto con exito y esta listo a ser leido - - Excepciones: - Si un determinado file no puede ser abierto - """ + Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si + no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files + siguientes. + + Affected: + self.flagIsNewFile + self.filename + self.fileSize + self.fp + self.set + self.flagNoMoreFiles + + Return: + 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado + 1 : si el file fue abierto con exito y esta listo a ser leido + + Excepciones: + Si un determinado file no puede ser abierto + """ + nFiles = 0 fileOk_flag = False firstTime_flag = True @@ -882,13 +883,34 @@ class JRODataReader(JRODataIO): def setNextFile(self): if self.fp != None: self.fp.close() - if self.online: newFile = self.__setNextFileOnline() else: newFile = self.__setNextFileOffline() - if not(newFile): + if self.onlineWithDate is True: + self.onlineWithDate=False + self.online = True + self.firstTime = False + self.setup( + path=self.path, + startDate=self.startDate, + endDate=self.endDate, + startTime=self.startTime , + endTime=self.endTime, + set=self.set, + expLabel=self.expLabel, + ext=self.ext, + online=self.online, + delay=self.delay, + walk=self.walk, + getblock=self.getblock, + nTxs=self.nTxs, + realtime=self.realtime, + blocksize=self.blocksize, + blocktime=self.blocktime + ) + return 1 print '[Reading] No more files to read' return 0 @@ -1046,12 +1068,9 @@ class JRODataReader(JRODataIO): if not(self.__setNewBlock()): print 'returning' return 0 - if not(self.readBlock()): return 0 - self.getBasicHeader() - if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime): print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks, @@ -1328,6 +1347,7 @@ class JRODataReader(JRODataIO): walk=walk, cursor=cursor, skip=skip, queue=queue) +<<<<<<< HEAD if not(pathList): # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path, # datetime.datetime.combine(startDate,startTime).ctime(), @@ -1340,6 +1360,129 @@ class JRODataReader(JRODataIO): self.filenameList = [] return +======= + if path == None: + raise ValueError, "[Reading] The path is not valid" + + + if ext == None: + ext = self.ext + + self.path = path + self.startDate = startDate + self.endDate = endDate + self.startTime = startTime + self.endTime = endTime + self.set = set + self.expLabel = expLabel + self.ext = ext + self.online = online + self.delay = delay + self.walk = walk + self.getblock = getblock + self.nTxs = nTxs + self.realtime = realtime + self.blocksize = blocksize + self.blocktime = blocktime + + + if self.firstTime is True: + pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate, + startTime=startTime, endTime=endTime, + set=set, expLabel=expLabel, ext=ext, + walk=walk) + filenameList = filenameList[:-1] + + if pathList is not None and filenameList is not None and online: + self.onlineWithDate = True + online = False + self.fileIndex = -1 + self.pathList = pathList + self.filenameList = filenameList + file_name = os.path.basename(filenameList[-1]) + basename, ext = os.path.splitext(file_name) + last_set = int(basename[-3:]) + + if online: + print "[Reading] Searching files in online mode..." + + for nTries in range(self.nTries): + fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, + expLabel=expLabel, + ext=ext, + walk=walk, + startDate=startDate, + startTime=startTime, + set=set) + + if fullpath: + break + print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1) + sleep( self.delay ) + + if not(fullpath): + print "[Reading] There 'isn't any valid file in %s" % path + return + + self.year = year + self.doy = doy + self.set = set - 1 + self.path = path + self.foldercounter = foldercounter + last_set = None + + else: + print "[Reading] Searching files in offline mode ..." + pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate, + startTime=startTime, endTime=endTime, + set=set, expLabel=expLabel, ext=ext, + walk=walk) + + if not(pathList): + # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path, + # datetime.datetime.combine(startDate,startTime).ctime(), + # datetime.datetime.combine(endDate,endTime).ctime()) + + # sys.exit(-1) + + self.fileIndex = -1 + self.pathList = [] + self.filenameList = [] + return + + self.fileIndex = -1 + self.pathList = pathList + self.filenameList = filenameList + file_name = os.path.basename(filenameList[-1]) + basename, ext = os.path.splitext(file_name) + last_set = int(basename[-3:]) + + + self.online = online + self.realtime = realtime + self.delay = delay + ext = ext.lower() + self.ext = ext + self.getByBlock = getblock + self.nTxs = nTxs + self.startTime = startTime + self.endTime = endTime + + + #Added----------------- + self.selBlocksize = blocksize + self.selBlocktime = blocktime + + + if not(self.setNextFile()): + if (startDate!=None) and (endDate!=None): + print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()) + elif startDate != None: + print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime()) + else: + print "[Reading] No files" + +>>>>>>> online_data_hour self.fileIndex = -1 self.pathList = pathList self.filenameList = filenameList @@ -1365,6 +1508,7 @@ class JRODataReader(JRODataIO): self.verbose = verbose self.warnings = warnings +<<<<<<< HEAD if not(self.setNextFile()): if (startDate!=None) and (endDate!=None): print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()) @@ -1383,6 +1527,13 @@ class JRODataReader(JRODataIO): if last_set != None: self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock return +======= + # self.getBasicHeader() + + if last_set != None: + self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock + return +>>>>>>> online_data_hour def getBasicHeader(self): @@ -1399,10 +1550,10 @@ class JRODataReader(JRODataIO): self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs - -# self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs - - + + # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs + + def getFirstHeader(self): raise NotImplementedError @@ -1476,6 +1627,7 @@ class JRODataReader(JRODataIO): verbose=True, **kwargs): if not(self.isConfig): +<<<<<<< HEAD # self.dataOut = dataOut self.setup( path=path, startDate=startDate, @@ -1499,6 +1651,11 @@ class JRODataReader(JRODataIO): warnings=warnings, server=server, verbose=verbose) +======= + + # self.dataOut = dataOut + self.setup(**kwargs) +>>>>>>> online_data_hour self.isConfig = True if server is None: self.getData() @@ -1621,9 +1778,15 @@ class JRODataWriter(JRODataIO): Return: None """ +<<<<<<< HEAD # CALCULAR PARAMETROS +======= + + # CALCULAR PARAMETROS + +>>>>>>> online_data_hour sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader diff --git a/schainpy/model/io/jroIO_voltage.py b/schainpy/model/io/jroIO_voltage.py index 6c6a23f..ecf11a8 100644 --- a/schainpy/model/io/jroIO_voltage.py +++ b/schainpy/model/io/jroIO_voltage.py @@ -180,13 +180,20 @@ class VoltageReader(JRODataReader, ProcessingUnit): def getBlockDimension(self): """ +<<<<<<< HEAD Obtiene la cantidad de puntos a leer por cada bloque de datos Affected: self.blocksize - - Return: - None +======= + Obtiene la cantidad de puntos a leer por cada bloque de datos + + Affected: + self.blocksize +>>>>>>> online_data_hour + + Return: + None """ pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels self.blocksize = pts2read @@ -195,6 +202,7 @@ class VoltageReader(JRODataReader, ProcessingUnit): def readBlock(self): """ +<<<<<<< HEAD readBlock lee el bloque de datos desde la posicion actual del puntero del archivo (self.fp) y actualiza todos los parametros relacionados al bloque de datos (metadata + data). La data leida es almacenada en el buffer y el contador del buffer @@ -215,6 +223,28 @@ class VoltageReader(JRODataReader, ProcessingUnit): Exceptions: Si un bloque leido no es un bloque valido +======= + readBlock lee el bloque de datos desde la posicion actual del puntero del archivo + (self.fp) y actualiza todos los parametros relacionados al bloque de datos + (metadata + data). La data leida es almacenada en el buffer y el contador del buffer + es seteado a 0 + + Inputs: + None + + Return: + None + + Affected: + self.profileIndex + self.datablock + self.flagIsNewFile + self.flagIsNewBlock + self.nTotalBlocks + + Exceptions: + Si un bloque leido no es un bloque valido +>>>>>>> online_data_hour """ # if self.server is not None: @@ -408,6 +438,7 @@ class VoltageReader(JRODataReader, ProcessingUnit): def getData(self): """ +<<<<<<< HEAD getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando @@ -441,6 +472,41 @@ class VoltageReader(JRODataReader, ProcessingUnit): self.profileIndex self.flagDiscontinuousBlock self.flagIsNewBlock +======= + getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut + del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos + en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando + "readNextBlock" + + Ademas incrementa el contador del buffer "self.profileIndex" en 1. + + Return: + + Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex + es igual al total de perfiles leidos desde el archivo. + + Si self.getByBlock == False: + + self.dataOut.data = buffer[:, thisProfile, :] + + shape = [nChannels, nHeis] + + Si self.getByBlock == True: + + self.dataOut.data = buffer[:, :, :] + + shape = [nChannels, nProfiles, nHeis] + + Variables afectadas: + self.dataOut + self.profileIndex + + Affected: + self.dataOut + self.profileIndex + self.flagDiscontinuousBlock + self.flagIsNewBlock +>>>>>>> online_data_hour """ if self.flagNoMoreFiles: self.dataOut.flagNoData = True @@ -462,16 +528,24 @@ class VoltageReader(JRODataReader, ProcessingUnit): if not self.getByBlock: """ +<<<<<<< HEAD Return profile by profile If nTxs > 1 then one profile is divided by nTxs and number of total blocks is increased by nTxs (nProfiles *= nTxs) +======= + Return profile by profile + + If nTxs > 1 then one profile is divided by nTxs and number of total + blocks is increased by nTxs (nProfiles *= nTxs) +>>>>>>> online_data_hour """ self.dataOut.flagDataAsBlock = False self.dataOut.data = self.datablock[:,self.profileIndex,:] self.dataOut.profileIndex = self.profileIndex self.profileIndex += 1 +<<<<<<< HEAD # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles: # """ @@ -483,9 +557,22 @@ class VoltageReader(JRODataReader, ProcessingUnit): # # self.profileIndex = self.dataOut.nProfiles +======= + + # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles: + # """ + # Return all block + # """ + # self.dataOut.flagDataAsBlock = True + # self.dataOut.data = self.datablock + # self.dataOut.profileIndex = self.dataOut.nProfiles - 1 + # + # self.profileIndex = self.dataOut.nProfiles + +>>>>>>> online_data_hour else: """ - Return a block + Return a block """ if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles if self.selBlocktime != None: diff --git a/schainpy/model/utils/jroutils_ftp.py b/schainpy/model/utils/jroutils_ftp.py index b72265e..b992d9a 100644 --- a/schainpy/model/utils/jroutils_ftp.py +++ b/schainpy/model/utils/jroutils_ftp.py @@ -166,12 +166,21 @@ class Remote(Thread): return 0 self.mutex.acquire() +<<<<<<< HEAD # init = time.time() # # while(self.bussy): # sleep(0.1) # if time.time() - init > 2*self.period: # return 0 +======= + # init = time.time() + # + # while(self.bussy): + # sleep(0.1) + # if time.time() - init > 2*self.period: + # return 0 +>>>>>>> online_data_hour self.fileList = fileList self.mutex.release() @@ -196,7 +205,11 @@ class Remote(Thread): if self.stopFlag: break +<<<<<<< HEAD # self.bussy = True +======= + # self.bussy = True +>>>>>>> online_data_hour self.mutex.acquire() print "[Remote Server] Opening %s" %self.__server @@ -619,6 +632,7 @@ class SendToServer(ProcessingUnit): filenameList = glob.glob1(thisFolder, '*%s' %self.ext) if len(filenameList) < 1: + continue for thisFile in filenameList: diff --git a/schainpy/scripts/JASMET30_MetDet.py b/schainpy/scripts/JASMET30_MetDet.py index e87355d..2ee8e12 100644 --- a/schainpy/scripts/JASMET30_MetDet.py +++ b/schainpy/scripts/JASMET30_MetDet.py @@ -55,17 +55,26 @@ opObj00.addParameter(name='channelList', value='0,1,2,3,4', format='intlist') opObj01 = procUnitConfObj0.addOperation(name='setRadarFrequency') opObj01.addParameter(name='frequency', value='30.e6', format='float') -opObj01 = procUnitConfObj0.addOperation(name='interpolateHeights') -opObj01.addParameter(name='topLim', value='73', format='int') -opObj01.addParameter(name='botLim', value='71', format='int') +# opObj01 = procUnitConfObj0.addOperation(name='interpolateHeights') +# opObj01.addParameter(name='topLim', value='73', format='int') +# opObj01.addParameter(name='botLim', value='71', format='int') opObj02 = procUnitConfObj0.addOperation(name='Decoder', optype='other') opObj03 = procUnitConfObj0.addOperation(name='CohInt', optype='other') opObj03.addParameter(name='n', value='2', format='int') +procUnitConfObj1 = controllerObj.addProcUnit(datatype='SpectraProc', inputId=procUnitConfObj0.getId()) +opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other') +opObj11.addParameter(name='id', value='237', format='int') +opObj11.addParameter(name='xmin', value='9.0', format='float') +opObj11.addParameter(name='xmax', value='16.0', format='float') +opObj11.addParameter(name='zmin', value='15.0', format='float') +opObj11.addParameter(name='zmax', value='50.0', format='float') + #--------------------------- Parameters Processing Unit ------------------------------------ +<<<<<<< HEAD procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj0.getId()) # opObj10 = procUnitConfObj1.addOperation(name='SMDetection', optype='other') @@ -83,3 +92,23 @@ opObj12.addParameter(name='mode',value='2',format='int') #-------------------------------------------------------------------------------------------------- controllerObj.start() +======= +# procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj0.getId()) +# # +# opObj10 = procUnitConfObj1.addOperation(name='SMDetection', optype='other') +# opObj10.addParameter(name='azimuth', value='45', format='float') +# opObj10.addParameter(name='hmin', value='60', format='float') +# opObj10.addParameter(name='hmax', value='120', format='float') + +# opObj12 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other') +# opObj12.addParameter(name='path', value=pathfile) +# opObj12.addParameter(name='blocksPerFile', value='1000', format='int') +# opObj12.addParameter(name='metadataList',value='type,heightList,paramInterval,timeZone',format='list') +# opObj12.addParameter(name='dataList',value='data_param,utctime',format='list') +# opObj12.addParameter(name='mode',value='2',format='int') + +#-------------------------------------------------------------------------------------------------- +controllerObj.createObjects() +controllerObj.connectObjects() +controllerObj.run() +>>>>>>> online_data_hour