import numpy import time import os import h5py import re from schainpy.model.data.jrodata import * from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation from schainpy.model.io.jroIO_base import * class HDF5Reader(ProcessingUnit): ext = ".hdf5" optchar = "D" timezone = None secStart = None secEnd = None fileIndex = None blockIndex = None blocksPerFile = None path = None #List of Files filenameList = None datetimeList = None #Hdf5 File fpMetadata = None pathMeta = None listMetaname = None listMeta = None listDataname = None listData = None listShapes = None fp = None #dataOut reconstruction dataOut = None nRecords = None def __init__(self): self.dataOut = self.__createObjByDefault() return def __createObjByDefault(self): dataObj = Parameters() return dataObj def setup(self,path=None, startDate=None, endDate=None, startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59), walk=True, timezone='ut', all=0, online=False, ext=None): if ext==None: ext = self.ext self.timezone = timezone # self.all = all # self.online = online self.path = path startDateTime = datetime.datetime.combine(startDate,startTime) endDateTime = datetime.datetime.combine(endDate,endTime) secStart = (startDateTime-datetime.datetime(1970,1,1)).total_seconds() secEnd = (endDateTime-datetime.datetime(1970,1,1)).total_seconds() self.secStart = secStart self.secEnd = secEnd if not(online): #Busqueda de archivos offline self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, secStart, secEnd, walk) else: self.__searchFilesOnline(path, walk) if not(self.filenameList): print "There is no files into the folder: %s"%(path) sys.exit(-1) # self.__getExpParameters() self.fileIndex = -1 self.__setNextFileOffline() self.__readMetadata() self.blockIndex = 0 return def __searchFilesOffline(self, path, startDate, endDate, ext, startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59), secStart = 0, secEnd = numpy.inf, walk=True): # self.__setParameters(path, startDate, endDate, startTime, endTime, walk) # # self.__checkPath() # # self.__findDataForDates() # # self.__selectDataForTimes() # # for i in range(len(self.filenameList)): # print "%s" %(self.filenameList[i]) pathList = [] if not walk: #pathList.append(path) multi_path = path.split(',') for single_path in multi_path: pathList.append(single_path) else: #dirList = [] multi_path = path.split(',') for single_path in multi_path: dirList = [] for thisPath in os.listdir(single_path): if not os.path.isdir(os.path.join(single_path,thisPath)): continue if not isDoyFolder(thisPath): continue dirList.append(thisPath) if not(dirList): return None, None thisDate = startDate while(thisDate <= endDate): year = thisDate.timetuple().tm_year doy = thisDate.timetuple().tm_yday matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*') if len(matchlist) == 0: thisDate += datetime.timedelta(1) continue for match in matchlist: pathList.append(os.path.join(single_path,match)) thisDate += datetime.timedelta(1) if pathList == []: print "Any folder was found for the date range: %s-%s" %(startDate, endDate) return None, None print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate) filenameList = [] datetimeList = [] pathDict = {} filenameList_to_sort = [] for i in range(len(pathList)): thisPath = pathList[i] fileList = glob.glob1(thisPath, "*%s" %ext) fileList.sort() pathDict.setdefault(fileList[0]) pathDict[fileList[0]] = i filenameList_to_sort.append(fileList[0]) filenameList_to_sort.sort() for file in filenameList_to_sort: thisPath = pathList[pathDict[file]] fileList = glob.glob1(thisPath, "*%s" %ext) fileList.sort() for file in fileList: filename = os.path.join(thisPath,file) thisDatetime = self.__isFileinThisTime(filename, secStart, secEnd) if not(thisDatetime): continue filenameList.append(filename) datetimeList.append(thisDatetime) if not(filenameList): print "Any file was found for the time range %s - %s" %(startTime, endTime) return None, None print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime) print for i in range(len(filenameList)): print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) self.filenameList = filenameList self.datetimeList = datetimeList return pathList, filenameList def __isFileinThisTime(self, filename, startSeconds, endSeconds): """ Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. Inputs: filename : nombre completo del archivo de datos en formato Jicamarca (.r) startTime : tiempo inicial del rango seleccionado en formato datetime.time endTime : tiempo final del rango seleccionado en formato datetime.time Return: Boolean : Retorna True si el archivo de datos contiene datos en el rango de fecha especificado, de lo contrario retorna False. Excepciones: Si el archivo no existe o no puede ser abierto Si la cabecera no puede ser leida. """ try: fp = fp = h5py.File(filename,'r') except IOError: traceback.print_exc() raise IOError, "The file %s can't be opened" %(filename) grp = fp['Data'] timeAux = grp['time'] time0 = timeAux[:][0].astype(numpy.float) #Time Vector fp.close() if self.timezone == 'lt': time0 -= 5*3600 boolTimer = numpy.logical_and(time0 >= startSeconds,time0 < endSeconds) if not (numpy.any(boolTimer)): return None thisDatetime = datetime.datetime.utcfromtimestamp(time0[0]) return thisDatetime def __checkPath(self): if os.path.exists(self.path): self.status = 1 else: self.status = 0 print 'Path:%s does not exists'%self.path return def __setNextFileOffline(self): idFile = self.fileIndex idFile += 1 if not(idFile < len(self.filenameList)): print "No more Files" return 0 filename = self.filenameList[idFile] filePointer = h5py.File(filename,'r') self.flagIsNewFile = 1 self.fileIndex = idFile self.filename = filename self.fp = filePointer print "Setting the file: %s"%self.filename self.__readMetadata() self.__setBlockList() # self.nRecords = self.fp['Data'].attrs['blocksPerFile'] self.nRecords = self.fp['Data'].attrs['nRecords'] self.blockIndex = 0 return 1 def __setBlockList(self): ''' self.fp self.startDateTime self.endDateTime self.blockList self.blocksPerFile ''' filePointer = self.fp secStart = self.secStart secEnd = self.secEnd grp = filePointer['Data'] timeVector = grp['time'].value.astype(numpy.float)[0] if self.timezone == 'lt': timeVector -= 5*3600 ind = numpy.where(numpy.logical_and(timeVector >= secStart , timeVector < secEnd))[0] self.blockList = ind self.blocksPerFile = len(ind) return def __readMetadata(self): ''' self.pathMeta self.listShapes self.listMetaname self.listMeta ''' grp = self.fp['Data'] pathMeta = os.path.join(self.path, grp.attrs['metadata']) if pathMeta == self.pathMeta: return else: self.pathMeta = pathMeta filePointer = h5py.File(self.pathMeta,'r') groupPointer = filePointer['Metadata'] listMetaname = [] listMetadata = [] for item in groupPointer.items(): name = item[0] if name=='array dimensions': table = groupPointer[name][:] listShapes = {} for shapes in table: listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4]]) else: data = groupPointer[name].value listMetaname.append(name) listMetadata.append(data) if name=='type': self.__initDataOut(data) filePointer.close() self.listShapes = listShapes self.listMetaname = listMetaname self.listMeta = listMetadata return def __readData(self): grp = self.fp['Data'] listdataname = [] listdata = [] for item in grp.items(): name = item[0] if name == 'time': listdataname.append('utctime') timeAux = grp[name].value.astype(numpy.float)[0] listdata.append(timeAux) continue listdataname.append(name) array = self.__setDataArray(self.nRecords, grp[name],self.listShapes[name]) listdata.append(array) self.listDataname = listdataname self.listData = listdata return def __setDataArray(self, nRecords, dataset, shapes): nChannels = shapes[0] #Dimension 0 nPoints = shapes[1] #Dimension 1, number of Points or Parameters nSamples = shapes[2] #Dimension 2, number of samples or ranges mode = shapes[3] # if nPoints>1: # arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples)) # else: # arrayData = numpy.zeros((nRecords,nChannels,nSamples)) # # chn = 'channel' # # for i in range(nChannels): # # data = dataset[chn + str(i)].value # # if nPoints>1: # data = numpy.rollaxis(data,2) # # arrayData[:,i,:] = data arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples)) doSqueeze = False if mode == 0: strds = 'channel' nDatas = nChannels newShapes = (nRecords,nPoints,nSamples) if nPoints == 1: doSqueeze = True axisSqueeze = 2 else: strds = 'param' nDatas = nPoints newShapes = (nRecords,nChannels,nSamples) if nChannels == 1: doSqueeze = True axisSqueeze = 1 for i in range(nDatas): data = dataset[strds + str(i)].value data = data.reshape(newShapes) if mode == 0: arrayData[:,i,:,:] = data else: arrayData[:,:,i,:] = data if doSqueeze: arrayData = numpy.squeeze(arrayData, axis=axisSqueeze) return arrayData def __initDataOut(self, type): # if type =='Parameters': # self.dataOut = Parameters() # elif type =='Spectra': # self.dataOut = Spectra() # elif type =='Voltage': # self.dataOut = Voltage() # elif type =='Correlation': # self.dataOut = Correlation() return def __setDataOut(self): listMeta = self.listMeta listMetaname = self.listMetaname listDataname = self.listDataname listData = self.listData blockIndex = self.blockIndex blockList = self.blockList for i in range(len(listMeta)): setattr(self.dataOut,listMetaname[i],listMeta[i]) for j in range(len(listData)): if listDataname[j]=='utctime': # setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex]]) setattr(self.dataOut,'utctimeInit',listData[j][blockList[blockIndex]]) continue setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex],:]) return self.dataOut.data_param def getData(self): # if self.flagNoMoreFiles: # self.dataOut.flagNoData = True # print 'Process finished' # return 0 # if self.blockIndex==self.blocksPerFile: if not( self.__setNextFileOffline() ): self.dataOut.flagNoData = True return 0 # # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers # self.dataOut.flagNoData = True # return 0 self.__readData() self.__setDataOut() self.dataOut.flagNoData = False self.blockIndex += 1 return def run(self, **kwargs): if not(self.isConfig): self.setup(**kwargs) # self.setObjProperties() self.isConfig = True self.getData() return class HDF5Writer(Operation): ext = ".hdf5" optchar = "D" metaoptchar = "M" metaFile = None filename = None path = None setFile = None fp = None grp = None ds = None firsttime = True #Configurations blocksPerFile = None blockIndex = None dataOut = None #Data Arrays dataList = None metadataList = None arrayDim = None tableDim = None # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')] dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')] mode = None nDatas = None #Number of datasets to be stored per array nDims = None #Number Dimensions in each dataset nDimsForDs = None def __init__(self): Operation.__init__(self) self.isConfig = False return def setup(self, dataOut, **kwargs): self.path = kwargs['path'] if kwargs.has_key('ext'): self.ext = kwargs['ext'] if kwargs.has_key('blocksPerFile'): self.blocksPerFile = kwargs['blocksPerFile'] else: self.blocksPerFile = 10 self.metadataList = kwargs['metadataList'] self.dataList = kwargs['dataList'] self.dataOut = dataOut if kwargs.has_key('mode'): mode = kwargs['mode'] if type(mode) == int: mode = numpy.zeros(len(self.dataList)) + mode else: mode = numpy.zeros(len(self.dataList)) self.mode = mode arrayDim = numpy.zeros((len(self.dataList),5)) #Table dimensions dtype0 = self.dtype tableList = [] for i in range(len(self.dataList)): dataAux = getattr(self.dataOut, self.dataList[i]) if type(dataAux)==float or type(dataAux)==int: arrayDim[i,0] = 1 else: arrayDim0 = dataAux.shape arrayDim[i,0] = len(arrayDim0) arrayDim[i,4] = mode[i] if len(arrayDim0) == 3: arrayDim[i,1:-1] = numpy.array(arrayDim0) elif len(arrayDim0) == 2: arrayDim[i,2:-1] = numpy.array(arrayDim0) #nHeights elif len(arrayDim0) == 1: arrayDim[i,3] = arrayDim0 elif len(arrayDim0) == 0: arrayDim[i,0] = 1 arrayDim[i,3] = 1 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0) tableList.append(table) self.arrayDim = arrayDim self.tableDim = numpy.array(tableList, dtype = dtype0) self.blockIndex = 0 return def putMetadata(self): fp = self.createMetadataFile() self.writeMetadata(fp) fp.close() return def createMetadataFile(self): ext = self.ext path = self.path setFile = self.setFile timeTuple = time.localtime(self.dataOut.utctime) subfolder = '' fullpath = os.path.join( path, subfolder ) if not( os.path.exists(fullpath) ): os.mkdir(fullpath) setFile = -1 #inicializo mi contador de seteo subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) fullpath = os.path.join( path, subfolder ) if not( os.path.exists(fullpath) ): os.mkdir(fullpath) setFile = -1 #inicializo mi contador de seteo else: filesList = os.listdir( fullpath ) filesList = sorted( filesList, key=str.lower ) if len( filesList ) > 0: filesList = [k for k in filesList if 'M' in k] filen = filesList[-1] # el filename debera tener el siguiente formato # 0 1234 567 89A BCDE (hex) # x YYYY DDD SSS .ext if isNumber( filen[8:11] ): setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file else: setFile = -1 else: setFile = -1 #inicializo mi contador de seteo setFile += 1 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext ) filename = os.path.join( path, subfolder, file ) self.metaFile = file #Setting HDF5 File fp = h5py.File(filename,'w') return fp def writeMetadata(self, fp): grp = fp.create_group("Metadata") grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype) for i in range(len(self.metadataList)): grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i])) return def setNextFile(self): ext = self.ext path = self.path setFile = self.setFile mode = self.mode timeTuple = time.localtime(self.dataOut.utctime) subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) fullpath = os.path.join( path, subfolder ) if os.path.exists(fullpath): filesList = os.listdir( fullpath ) filesList = [k for k in filesList if 'D' in k] if len( filesList ) > 0: filesList = sorted( filesList, key=str.lower ) filen = filesList[-1] # el filename debera tener el siguiente formato # 0 1234 567 89A BCDE (hex) # x YYYY DDD SSS .ext if isNumber( filen[8:11] ): setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file else: setFile = -1 else: setFile = -1 #inicializo mi contador de seteo setFile += 1 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext ) filename = os.path.join( path, subfolder, file ) #Setting HDF5 File fp = h5py.File(filename,'w') grp = fp.create_group("Data") grp.attrs['metadata'] = self.metaFile # grp.attrs['blocksPerFile'] = 0 ds = [] data = [] nDimsForDs = [] nDatas = numpy.zeros(len(self.dataList)) nDims = self.arrayDim[:,0] nDim1 = self.arrayDim[:,2] nDim0 = self.arrayDim[:,3] for i in range(len(self.dataList)): if nDims[i]==1: # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20') ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64) ds.append(ds0) data.append([]) nDimsForDs.append(nDims[i]) else: if mode[i]==0: strMode = "channel" nDatas[i] = self.arrayDim[i,1] else: strMode = "param" nDatas[i] = self.arrayDim[i,2] if nDims[i]==2: nDatas[i] = self.arrayDim[i,2] grp0 = grp.create_group(self.dataList[i]) for j in range(int(nDatas[i])): tableName = strMode + str(j) if nDims[i] == 3: ds0 = grp0.create_dataset(tableName, (nDim1[i],nDim0[i],1) , data = numpy.zeros((nDim1[i],nDim0[i],1)) ,maxshape=(None,nDim0[i],None), chunks=True) else: ds0 = grp0.create_dataset(tableName, (1,nDim0[i]), data = numpy.zeros((1,nDim0[i])) , maxshape=(None,nDim0[i]), chunks=True) ds.append(ds0) data.append([]) nDimsForDs.append(nDims[i]) self.nDatas = nDatas self.nDims = nDims self.nDimsForDs = nDimsForDs #Saving variables print 'Writing the file: %s'%filename self.filename = filename self.fp = fp self.grp = grp self.grp.attrs.modify('nRecords', 1) self.ds = ds self.data = data self.setFile = setFile self.firsttime = True self.blockIndex = 0 return def putData(self): if not self.firsttime: self.readBlock() if self.blockIndex == self.blocksPerFile: self.setNextFile() self.setBlock() self.writeBlock() self.fp.flush() self.fp.close() return def readBlock(self): ''' data Array configured self.data ''' ds = self.ds #Setting HDF5 File fp = h5py.File(self.filename,'r+') grp = fp["Data"] ind = 0 # grp.attrs['blocksPerFile'] = 0 for i in range(len(self.dataList)): if self.nDims[i]==1: ds0 = grp[self.dataList[i]] ds[ind] = ds0 ind += 1 else: if self.mode[i]==0: strMode = "channel" else: strMode = "param" grp0 = grp[self.dataList[i]] for j in range(int(self.nDatas[i])): tableName = strMode + str(j) ds0 = grp0[tableName] ds[ind] = ds0 ind += 1 self.fp = fp self.grp = grp self.ds = ds return def setBlock(self): ''' data Array configured self.data ''' #Creating Arrays data = self.data nDatas = self.nDatas nDims = self.nDims mode = self.mode ind = 0 for i in range(len(self.dataList)): dataAux = getattr(self.dataOut,self.dataList[i]) if nDims[i] == 1: # data[ind] = numpy.array([str(dataAux)]).reshape((1,1)) data[ind] = dataAux # if not self.firsttime: # data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind])) ind += 1 else: for j in range(int(nDatas[i])): if (mode[i] == 0) or (nDims[i] == 2): #In case division per channel or Dimensions is only 1 data[ind] = dataAux[j,:] else: data[ind] = dataAux[:,j,:] # if nDims[i] == 3: # data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1)) # if not self.firsttime: # data[ind] = numpy.dstack((self.ds[ind][:], data[ind])) # else: # data[ind] = data[ind].reshape((1,data[ind].shape[0])) # if not self.firsttime: # data[ind] = numpy.vstack((self.ds[ind][:], data[ind])) ind += 1 self.data = data return def writeBlock(self): ''' Saves the block in the HDF5 file ''' for i in range(len(self.ds)): if self.firsttime: # self.ds[i].resize(self.data[i].shape) # self.ds[i][self.blockIndex,:] = self.data[i] if type(self.data[i]) == numpy.ndarray: nDims1 = len(self.ds[i].shape) if nDims1 == 3: self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1)) self.ds[i].resize(self.data[i].shape) self.ds[i][:] = self.data[i] else: if self.nDimsForDs[i] == 1: self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1)) self.ds[i][0,-1] = self.data[i] elif self.nDimsForDs[i] == 2: self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1])) self.ds[i][self.blockIndex,:] = self.data[i] elif self.nDimsForDs[i] == 3: dataShape = self.data[i].shape dsShape = self.ds[i].shape if dataShape[0]==dsShape[0]: self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1)) self.ds[i][:,:,-1] = self.data[i] else: self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1],self.ds[i].shape[2])) self.ds[i][dsShape[0]:,:,0] = self.data[i] # self.ds[i].append(self.data[i]) # self.fp.flush() # if not self.firsttime: # self.fp.root.Data._v_attrs.nRecords = self.blockIndex # if self.firsttime: # self.fp.close() # self.readBlock2() self.blockIndex += 1 self.firsttime = False return def run(self, dataOut, **kwargs): if not(self.isConfig): self.setup(dataOut, **kwargs) self.isConfig = True self.putMetadata() self.setNextFile() self.putData() return