diff --git a/schainpy/model/data/jrodata.py b/schainpy/model/data/jrodata.py index bd3b703..dbaea8f 100644 --- a/schainpy/model/data/jrodata.py +++ b/schainpy/model/data/jrodata.py @@ -1234,11 +1234,11 @@ class PlotterData(object): if plot == 'snr': buffer = 10*numpy.log10(dataOut.data_SNR) if plot == 'dop': - buffer = 10*numpy.log10(dataOut.data_DOP) - if plot == 'mean': - buffer = dataOut.data_MEAN - if plot == 'std': - buffer = dataOut.data_STD + buffer = dataOut.data_DOP + if plot == 'pow': + buffer = 10*numpy.log10(dataOut.data_POW) + if plot == 'width': + buffer = dataOut.data_WIDTH if plot == 'coh': buffer = dataOut.getCoherence() if plot == 'phase': diff --git a/schainpy/model/io/jroIO_base.py b/schainpy/model/io/jroIO_base.py index 1df7a64..69529ad 100644 --- a/schainpy/model/io/jroIO_base.py +++ b/schainpy/model/io/jroIO_base.py @@ -1205,7 +1205,7 @@ class JRODataReader(JRODataIO): thisDate = getDateFromRadarFile(thisFile) - if thisDate in dateList: + if thisDate in dateList or single_path in pathList: continue dateList.append(thisDate) @@ -1599,10 +1599,10 @@ class JRODataWriter(JRODataIO): self.basicHeaderObj.size = self.basicHeaderSize # bytes self.basicHeaderObj.version = self.versionFile self.basicHeaderObj.dataBlock = self.nTotalBlocks - + log.warning(datetime.datetime.fromtimestamp(self.dataOut.utctime)) utc = numpy.floor(self.dataOut.utctime) milisecond = (self.dataOut.utctime - utc) * 1000.0 - + log.warning(milisecond) self.basicHeaderObj.utc = utc self.basicHeaderObj.miliSecond = milisecond self.basicHeaderObj.timeZone = self.dataOut.timeZone diff --git a/schainpy/model/io/jroIO_param.py b/schainpy/model/io/jroIO_param.py index 505403e..bb3520c 100644 --- a/schainpy/model/io/jroIO_param.py +++ b/schainpy/model/io/jroIO_param.py @@ -7,74 +7,44 @@ import datetime from schainpy.model.data.jrodata import * from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator -# from .jroIO_base import * from schainpy.model.io.jroIO_base import * -import schainpy from schainpy.utils import log @MPDecorator class ParamReader(JRODataReader,ProcessingUnit): ''' Reads HDF5 format files - path - startDate - endDate - startTime - endTime ''' ext = ".hdf5" - optchar = "D" - timezone = None - startTime = None - endTime = None - fileIndex = None - utcList = None #To select data in the utctime list - blockList = None #List to blocks to be read from the file - blocksPerFile = None #Number of blocks to be read - blockIndex = None - path = None - #List of Files - filenameList = None - datetimeList = None - #Hdf5 File - listMetaname = None - listMeta = None - listDataname = None - listData = None - listShapes = None - fp = None - #dataOut reconstruction - dataOut = None - def __init__(self):#, **kwargs): ProcessingUnit.__init__(self) #, **kwargs) self.dataOut = Parameters() @@ -152,7 +122,6 @@ class ParamReader(JRODataReader,ProcessingUnit): #---------------------------------------------------------------------------------- for thisPath in pathList: - # thisPath = pathList[pathDict[file]] fileList = glob.glob1(thisPath, "*%s" %ext) fileList.sort() @@ -191,13 +160,9 @@ class ParamReader(JRODataReader,ProcessingUnit): Inputs: filename : nombre completo del archivo de datos en formato Jicamarca (.r) - startDate : fecha inicial del rango seleccionado en formato datetime.date - endDate : fecha final del rango seleccionado en formato datetime.date - startTime : tiempo inicial del rango seleccionado en formato datetime.time - endTime : tiempo final del rango seleccionado en formato datetime.time Return: @@ -217,7 +182,7 @@ class ParamReader(JRODataReader,ProcessingUnit): except IOError: traceback.print_exc() raise IOError("The file %s can't be opened" %(filename)) - #chino rata + #In case has utctime attribute grp2 = grp1['utctime'] # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time @@ -229,7 +194,6 @@ class ParamReader(JRODataReader,ProcessingUnit): thisUtcTime -= 5*3600 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600) - # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0]) thisDate = thisDatetime.date() thisTime = thisDatetime.time() @@ -269,24 +233,18 @@ class ParamReader(JRODataReader,ProcessingUnit): idFile = self.fileIndex if not(idFile < len(self.filenameList)): - print("No more Files") + self.dataOut.error = "No more Files" return 0 filename = self.filenameList[idFile] - filePointer = h5py.File(filename,'r') - self.filename = filename - self.fp = filePointer print("Setting the file: %s"%self.filename) - # self.__readMetadata() self.__setBlockList() self.__readData() - # self.nRecords = self.fp['Data'].attrs['blocksPerFile'] - # self.nRecords = self.fp['Data'].attrs['nRecords'] self.blockIndex = 0 return 1 @@ -333,28 +291,14 @@ class ParamReader(JRODataReader,ProcessingUnit): Reads Metadata self.pathMeta - self.listShapes self.listMetaname self.listMeta ''' - # grp = self.fp['Data'] - # pathMeta = os.path.join(self.path, grp.attrs['metadata']) - # - # if pathMeta == self.pathMeta: - # return - # else: - # self.pathMeta = pathMeta - # - # filePointer = h5py.File(self.pathMeta,'r') - # groupPointer = filePointer['Metadata'] - filename = self.filenameList[0] - fp = h5py.File(filename,'r') - gp = fp['Metadata'] listMetaname = [] @@ -372,9 +316,6 @@ class ParamReader(JRODataReader,ProcessingUnit): listMetaname.append(name) listMetadata.append(data) - # if name=='type': - # self.__initDataOut(data) - self.listShapes = listShapes self.listMetaname = listMetaname self.listMeta = listMetadata @@ -401,17 +342,11 @@ class ParamReader(JRODataReader,ProcessingUnit): def __setDataArray(self, dataset, shapes): nDims = shapes[0] - nDim2 = shapes[1] #Dimension 0 - nDim1 = shapes[2] #Dimension 1, number of Points or Parameters - nDim0 = shapes[3] #Dimension 2, number of samples or ranges - mode = shapes[4] #Mode of storing - blockList = self.blockList - blocksPerFile = self.blocksPerFile #Depending on what mode the data was stored @@ -517,7 +452,6 @@ class ParamReader(JRODataReader,ProcessingUnit): if not(self.isConfig): self.setup(**kwargs) - # self.setObjProperties() self.isConfig = True self.getData() @@ -530,18 +464,12 @@ class ParamWriter(Operation): HDF5 Writer, stores parameters data in HDF5 format files path: path where the files will be stored - blocksPerFile: number of blocks that will be saved in per HDF5 format file - mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors) - metadataList: list of attributes that will be stored as metadata - dataList: list of attributes that will be stores as data - ''' - ext = ".hdf5" optchar = "D" metaoptchar = "M" @@ -1034,4 +962,583 @@ class ParamWriter(Operation): self.putData() return - \ No newline at end of file + + +@MPDecorator +class ParameterReader(JRODataReader,ProcessingUnit): + ''' + Reads HDF5 format files + ''' + + ext = ".hdf5" + optchar = "D" + timezone = None + startTime = None + endTime = None + fileIndex = None + blockList = None #List to blocks to be read from the file + blocksPerFile = None #Number of blocks to be read + blockIndex = None + path = None + #List of Files + filenameList = None + datetimeList = None + #Hdf5 File + listMetaname = None + listMeta = None + listDataname = None + listData = None + listShapes = None + fp = None + #dataOut reconstruction + dataOut = None + + def __init__(self): + ProcessingUnit.__init__(self) + self.dataOut = Parameters() + return + + def setup(self, **kwargs): + + path = kwargs['path'] + startDate = kwargs['startDate'] + endDate = kwargs['endDate'] + startTime = kwargs['startTime'] + endTime = kwargs['endTime'] + walk = kwargs['walk'] + if 'ext' in kwargs: + ext = kwargs['ext'] + else: + ext = '.hdf5' + if 'timezone' in kwargs: + self.timezone = kwargs['timezone'] + else: + self.timezone = 'lt' + + print("[Reading] Searching files in offline mode ...") + pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate, + startTime=startTime, endTime=endTime, + ext=ext, walk=walk) + + if not(filenameList): + print("There is no files into the folder: %s"%(path)) + sys.exit(-1) + + self.fileIndex = -1 + self.startTime = startTime + self.endTime = endTime + self.__readMetadata() + self.__setNextFileOffline() + + return + + def searchFilesOffLine(self, path, startDate=None, endDate=None, startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59), ext='.hdf5', walk=True): + + expLabel = '' + self.filenameList = [] + self.datetimeList = [] + pathList = [] + dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True) + + if dateList == []: + print("[Reading] No *%s files in %s from %s to %s)"%(ext, path, + datetime.datetime.combine(startDate,startTime).ctime(), + datetime.datetime.combine(endDate,endTime).ctime())) + + return None, None + + if len(dateList) > 1: + print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)) + else: + print("[Reading] data was found for the date %s" %(dateList[0])) + + filenameList = [] + datetimeList = [] + + for thisPath in pathList: + + fileList = glob.glob1(thisPath, "*%s" %ext) + fileList.sort() + + for file in fileList: + + filename = os.path.join(thisPath,file) + + if not isFileInDateRange(filename, startDate, endDate): + continue + + thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime) + + if not(thisDatetime): + continue + + filenameList.append(filename) + datetimeList.append(thisDatetime) + + if not(filenameList): + print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())) + return None, None + + print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)) + print() + + self.filenameList = filenameList + self.datetimeList = datetimeList + + return pathList, filenameList + + def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime): + + """ + Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. + + Inputs: + filename : nombre completo del archivo de datos en formato Jicamarca (.r) + startDate : fecha inicial del rango seleccionado en formato datetime.date + endDate : fecha final del rango seleccionado en formato datetime.date + startTime : tiempo inicial del rango seleccionado en formato datetime.time + endTime : tiempo final del rango seleccionado en formato datetime.time + + Return: + Boolean : Retorna True si el archivo de datos contiene datos en el rango de + fecha especificado, de lo contrario retorna False. + + Excepciones: + Si el archivo no existe o no puede ser abierto + Si la cabecera no puede ser leida. + + """ + + try: + fp = h5py.File(filename, 'r') + grp1 = fp['Data'] + + except IOError: + traceback.print_exc() + raise IOError("The file %s can't be opened" %(filename)) + #In case has utctime attribute + grp2 = grp1['utctime'] + thisUtcTime = grp2.value[0] + + fp.close() + + if self.timezone == 'lt': + thisUtcTime -= 5*3600 + + thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime + 5*3600) + thisDate = thisDatetime.date() + thisTime = thisDatetime.time() + + startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds() + endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds() + + #General case + # o>>>>>>>>>>>>>><<<<<<<<<<<<<= startTime: + thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime) + if numpy.any(thisUtcLog): #If there is one block between the hours mentioned + return thisDatetime + return None + + #If endTime < startTime then endTime belongs to the next day + #<<<<<<<<<<>>>>>>>>>> + #-----------o----------------------------o----------- + # endTime startTime + + if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime): + return None + + if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime): + return None + + if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime): + return None + + return thisDatetime + + def __setNextFileOffline(self): + + self.fileIndex += 1 + idFile = self.fileIndex + + if not(idFile < len(self.filenameList)): + self.dataOut.error = 'No more files' + return 0 + + filename = self.filenameList[idFile] + self.fp = h5py.File(filename, 'r') + self.filename = filename + + print("Setting the file: %s"%self.filename) + + self.__setBlockList() + self.__readData() + self.blockIndex = 0 + return 1 + + def __setBlockList(self): + ''' + Selects the data within the times defined + + self.fp + self.startTime + self.endTime + self.blockList + self.blocksPerFile + + ''' + fp = self.fp + startTime = self.startTime + endTime = self.endTime + + grp = fp['Data'] + thisUtcTime = grp['utctime'].value.astype(numpy.float)[0] + + if self.timezone == 'lt': + thisUtcTime -= 5*3600 + + thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime + 5*3600) + + thisDate = thisDatetime.date() + thisTime = thisDatetime.time() + + startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds() + endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds() + + ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0] + + self.blockList = ind + self.blocksPerFile = len(ind) + + return + + def __readMetadata(self): + ''' + Reads Metadata + ''' + + filename = self.filenameList[0] + fp = h5py.File(filename, 'r') + gp = fp['Metadata'] + listMetaname = [] + listMetadata = [] + + for item in list(gp.items()): + name = item[0] + + if name=='variables': + table = gp[name][:] + listShapes = {} + for shapes in table: + listShapes[shapes[0].decode()] = numpy.array([shapes[1]]) + else: + data = gp[name].value + listMetaname.append(name) + listMetadata.append(data) + + self.listShapes = listShapes + self.listMetaname = listMetaname + self.listMeta = listMetadata + + fp.close() + return + + def __readData(self): + + grp = self.fp['Data'] + listdataname = [] + listdata = [] + + for item in list(grp.items()): + name = item[0] + listdataname.append(name) + dim = self.listShapes[name][0] + if dim == 0: + array = grp[name].value + else: + array = [] + for i in range(dim): + array.append(grp[name]['table{:02d}'.format(i)].value) + array = numpy.array(array) + + listdata.append(array) + + self.listDataname = listdataname + self.listData = listdata + return + + def getData(self): + + for i in range(len(self.listMeta)): + setattr(self.dataOut, self.listMetaname[i], self.listMeta[i]) + + for j in range(len(self.listData)): + dim = self.listShapes[self.listDataname[j]][0] + if dim == 0: + setattr(self.dataOut, self.listDataname[j], self.listData[j][self.blockIndex]) + else: + setattr(self.dataOut, self.listDataname[j], self.listData[j][:,self.blockIndex]) + + self.dataOut.flagNoData = False + self.blockIndex += 1 + + return + + def run(self, **kwargs): + + if not(self.isConfig): + self.setup(**kwargs) + self.isConfig = True + + if self.blockIndex == self.blocksPerFile: + if not(self.__setNextFileOffline()): + self.dataOut.flagNoData = True + return 0 + + self.getData() + + return + +@MPDecorator +class ParameterWriter(Operation): + ''' + HDF5 Writer, stores parameters data in HDF5 format files + + path: path where the files will be stored + blocksPerFile: number of blocks that will be saved in per HDF5 format file + mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors) + metadataList: list of attributes that will be stored as metadata + dataList: list of attributes that will be stores as data + ''' + + + ext = ".hdf5" + optchar = "D" + metaoptchar = "M" + metaFile = None + filename = None + path = None + setFile = None + fp = None + grp = None + ds = None + firsttime = True + #Configurations + blocksPerFile = None + blockIndex = None + dataOut = None + #Data Arrays + dataList = None + metadataList = None + dsList = None #List of dictionaries with dataset properties + tableDim = None + dtype = [('name', 'S20'),('nDim', 'i')] + currentDay = None + lastTime = None + + def __init__(self): + + Operation.__init__(self) + return + + def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None): + self.path = path + self.blocksPerFile = blocksPerFile + self.metadataList = metadataList + self.dataList = dataList + self.setType = setType + + tableList = [] + dsList = [] + + for i in range(len(self.dataList)): + dsDict = {} + dataAux = getattr(self.dataOut, self.dataList[i]) + dsDict['variable'] = self.dataList[i] + + if dataAux is None: + continue + elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)): + dsDict['nDim'] = 0 + else: + dsDict['nDim'] = len(dataAux.shape) + dsDict['shape'] = dataAux.shape + dsDict['dsNumber'] = dataAux.shape[0] + + dsList.append(dsDict) + tableList.append((self.dataList[i], dsDict['nDim'])) + + self.dsList = dsList + self.tableDim = numpy.array(tableList, dtype=self.dtype) + self.currentDay = self.dataOut.datatime.date() + + def timeFlag(self): + currentTime = self.dataOut.utctime + timeTuple = time.localtime(currentTime) + dataDay = timeTuple.tm_yday + + if self.lastTime is None: + self.lastTime = currentTime + self.currentDay = dataDay + return False + + timeDiff = currentTime - self.lastTime + + #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora + if dataDay != self.currentDay: + self.currentDay = dataDay + return True + elif timeDiff > 3*60*60: + self.lastTime = currentTime + return True + else: + self.lastTime = currentTime + return False + + def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, setType=None): + + self.dataOut = dataOut + if not(self.isConfig): + self.setup(path=path, blocksPerFile=blocksPerFile, + metadataList=metadataList, dataList=dataList, + setType=setType) + + self.isConfig = True + self.setNextFile() + + self.putData() + return + + def setNextFile(self): + + ext = self.ext + path = self.path + setFile = self.setFile + + timeTuple = time.localtime(self.dataOut.utctime) + subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) + fullpath = os.path.join(path, subfolder) + + if os.path.exists(fullpath): + filesList = os.listdir(fullpath) + filesList = [k for k in filesList if k.startswith(self.optchar)] + if len( filesList ) > 0: + filesList = sorted(filesList, key=str.lower) + filen = filesList[-1] + # el filename debera tener el siguiente formato + # 0 1234 567 89A BCDE (hex) + # x YYYY DDD SSS .ext + if isNumber(filen[8:11]): + setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file + else: + setFile = -1 + else: + setFile = -1 #inicializo mi contador de seteo + else: + os.makedirs(fullpath) + setFile = -1 #inicializo mi contador de seteo + + if self.setType is None: + setFile += 1 + file = '%s%4.4d%3.3d%03d%s' % (self.optchar, + timeTuple.tm_year, + timeTuple.tm_yday, + setFile, + ext ) + else: + setFile = timeTuple.tm_hour*60+timeTuple.tm_min + file = '%s%4.4d%3.3d%04d%s' % (self.optchar, + timeTuple.tm_year, + timeTuple.tm_yday, + setFile, + ext ) + + self.filename = os.path.join( path, subfolder, file ) + + #Setting HDF5 File + self.fp = h5py.File(self.filename, 'w') + #write metadata + self.writeMetadata(self.fp) + #Write data + self.writeData(self.fp) + + def writeMetadata(self, fp): + + grp = fp.create_group("Metadata") + grp.create_dataset('variables', data=self.tableDim, dtype=self.dtype) + + for i in range(len(self.metadataList)): + if not hasattr(self.dataOut, self.metadataList[i]): + log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name) + continue + value = getattr(self.dataOut, self.metadataList[i]) + grp.create_dataset(self.metadataList[i], data=value) + return + + def writeData(self, fp): + + grp = fp.create_group("Data") + dtsets = [] + data = [] + + for dsInfo in self.dsList: + if dsInfo['nDim'] == 0: + ds = grp.create_dataset( + dsInfo['variable'], + (self.blocksPerFile, ), + chunks=True, + dtype=numpy.float64) + dtsets.append(ds) + data.append((dsInfo['variable'], -1)) + else: + sgrp = grp.create_group(dsInfo['variable']) + for i in range(dsInfo['dsNumber']): + ds = sgrp.create_dataset( + 'table{:02d}'.format(i), + (self.blocksPerFile, ) + dsInfo['shape'][1:], + chunks=True) + dtsets.append(ds) + data.append((dsInfo['variable'], i)) + fp.flush() + + log.log('creating file: {}'.format(fp.filename), 'Writing') + + self.ds = dtsets + self.data = data + self.firsttime = True + self.blockIndex = 0 + return + + def putData(self): + + if (self.blockIndex == self.blocksPerFile) or self.timeFlag(): + self.closeFile() + self.setNextFile() + + for i, ds in enumerate(self.ds): + attr, ch = self.data[i] + if ch == -1: + ds[self.blockIndex] = getattr(self.dataOut, attr) + else: + ds[self.blockIndex] = getattr(self.dataOut, attr)[ch] + + self.fp.flush() + self.blockIndex += 1 + + return + + def closeFile(self): + + if self.blockIndex != self.blocksPerFile: + for ds in self.ds: + ds.resize(self.blockIndex, axis=0) + + self.fp.flush() + self.fp.close() + + def close(self): + + self.closeFile()