jroIO_HDF5.py
1201 lines
| 38.2 KiB
| text/x-python
|
PythonLexer
Julio Valdez
|
r514 | import numpy | ||
| import time | ||||
| import os | ||||
| import h5py | ||||
| import re | ||||
|
Julio Valdez
|
r804 | import datetime | ||
|
Julio Valdez
|
r514 | |||
Miguel Valdez
|
r568 | from schainpy.model.data.jrodata import * | ||
| from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation | ||||
|
Julio Valdez
|
r804 | # from jroIO_base import * | ||
|
Miguel Valdez
|
r568 | from schainpy.model.io.jroIO_base import * | ||
|
Julio Valdez
|
r804 | import schainpy | ||
|
Julio Valdez
|
r514 | |||
| class HDF5Reader(ProcessingUnit): | ||||
|
Julio Valdez
|
r804 | ''' | ||
| Reads HDF5 format files | ||||
| path | ||||
| startDate | ||||
| endDate | ||||
| startTime | ||||
| endTime | ||||
| ''' | ||||
|
Julio Valdez
|
r514 | |||
| ext = ".hdf5" | ||||
| optchar = "D" | ||||
| timezone = None | ||||
|
Julio Valdez
|
r804 | startTime = None | ||
|
Julio Valdez
|
r543 | |||
|
Julio Valdez
|
r804 | endTime = None | ||
|
Julio Valdez
|
r543 | |||
|
Julio Valdez
|
r514 | fileIndex = None | ||
|
Julio Valdez
|
r804 | utcList = None #To select data in the utctime list | ||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r804 | blockList = None #List to blocks to be read from the file | ||
| blocksPerFile = None #Number of blocks to be read | ||||
| blockIndex = None | ||||
|
Julio Valdez
|
r543 | |||
|
Julio Valdez
|
r514 | path = None | ||
|
Julio Valdez
|
r543 | #List of Files | ||
| filenameList = None | ||||
| datetimeList = None | ||||
|
Julio Valdez
|
r514 | #Hdf5 File | ||
| listMetaname = None | ||||
|
Julio Valdez
|
r543 | listMeta = None | ||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r543 | listDataname = None | ||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r543 | listData = None | ||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r543 | listShapes = None | ||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r543 | fp = None | ||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r543 | #dataOut reconstruction | ||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r543 | dataOut = None | ||
|
Julio Valdez
|
r514 | |||
| def __init__(self): | ||||
|
Julio Valdez
|
r804 | self.dataOut = Parameters() | ||
|
Julio Valdez
|
r514 | return | ||
|
Julio Valdez
|
r804 | def setup(self, **kwargs): | ||
| path = kwargs['path'] | ||||
| startDate = kwargs['startDate'] | ||||
| endDate = kwargs['endDate'] | ||||
| startTime = kwargs['startTime'] | ||||
| endTime = kwargs['endTime'] | ||||
| walk = kwargs['walk'] | ||||
| if kwargs.has_key('ext'): | ||||
| ext = kwargs['ext'] | ||||
| else: | ||||
| ext = '.hdf5' | ||||
| print "[Reading] Searching files in offline mode ..." | ||||
| pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate, | ||||
| startTime=startTime, endTime=endTime, | ||||
| ext=ext, walk=walk) | ||||
| if not(filenameList): | ||||
|
Julio Valdez
|
r514 | print "There is no files into the folder: %s"%(path) | ||
| sys.exit(-1) | ||||
| self.fileIndex = -1 | ||||
|
Julio Valdez
|
r804 | self.startTime = startTime | ||
| self.endTime = endTime | ||||
|
Julio Valdez
|
r514 | |||
| self.__readMetadata() | ||||
|
Julio Valdez
|
r804 | self.__setNextFileOffline() | ||
|
Julio Valdez
|
r514 | |||
| return | ||||
|
Julio Valdez
|
r804 | |||
| def __searchFilesOffLine(self, | ||||
|
Julio Valdez
|
r514 | path, | ||
|
Julio Valdez
|
r804 | startDate=None, | ||
| endDate=None, | ||||
|
Julio Valdez
|
r514 | startTime=datetime.time(0,0,0), | ||
| endTime=datetime.time(23,59,59), | ||||
|
Julio Valdez
|
r804 | ext='.hdf5', | ||
|
Julio Valdez
|
r514 | walk=True): | ||
|
Julio Valdez
|
r804 | expLabel = '' | ||
| self.filenameList = [] | ||||
| self.datetimeList = [] | ||||
|
Julio Valdez
|
r514 | |||
| pathList = [] | ||||
|
Julio Valdez
|
r804 | JRODataObj = JRODataReader() | ||
| dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True) | ||||
| if dateList == []: | ||||
| print "[Reading] No *%s files in %s from %s to %s)"%(ext, path, | ||||
| datetime.datetime.combine(startDate,startTime).ctime(), | ||||
| datetime.datetime.combine(endDate,endTime).ctime()) | ||||
|
Julio Valdez
|
r514 | |||
| return None, None | ||||
|
Julio Valdez
|
r804 | if len(dateList) > 1: | ||
| print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate) | ||||
| else: | ||||
| print "[Reading] data was found for the date %s" %(dateList[0]) | ||||
|
Julio Valdez
|
r514 | filenameList = [] | ||
| datetimeList = [] | ||||
|
Julio Valdez
|
r804 | #---------------------------------------------------------------------------------- | ||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r804 | for thisPath in pathList: | ||
| # thisPath = pathList[pathDict[file]] | ||||
|
Julio Valdez
|
r514 | |||
| fileList = glob.glob1(thisPath, "*%s" %ext) | ||||
| fileList.sort() | ||||
| for file in fileList: | ||||
| filename = os.path.join(thisPath,file) | ||||
|
Julio Valdez
|
r804 | |||
| if not isFileInDateRange(filename, startDate, endDate): | ||||
| continue | ||||
| thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime) | ||||
|
Julio Valdez
|
r514 | |||
| if not(thisDatetime): | ||||
| continue | ||||
| filenameList.append(filename) | ||||
| datetimeList.append(thisDatetime) | ||||
| if not(filenameList): | ||||
|
Julio Valdez
|
r804 | print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()) | ||
|
Julio Valdez
|
r514 | return None, None | ||
|
Julio Valdez
|
r804 | print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime) | ||
|
Julio Valdez
|
r514 | |||
| for i in range(len(filenameList)): | ||||
|
Julio Valdez
|
r804 | print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) | ||
|
Julio Valdez
|
r514 | |||
| self.filenameList = filenameList | ||||
| self.datetimeList = datetimeList | ||||
|
Julio Valdez
|
r804 | |||
|
Julio Valdez
|
r514 | return pathList, filenameList | ||
|
Julio Valdez
|
r804 | |||
| def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime): | ||||
|
Julio Valdez
|
r514 | """ | ||
| Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. | ||||
| Inputs: | ||||
| filename : nombre completo del archivo de datos en formato Jicamarca (.r) | ||||
|
Julio Valdez
|
r804 | startDate : fecha inicial del rango seleccionado en formato datetime.date | ||
| endDate : fecha final del rango seleccionado en formato datetime.date | ||||
|
Julio Valdez
|
r514 | startTime : tiempo inicial del rango seleccionado en formato datetime.time | ||
| endTime : tiempo final del rango seleccionado en formato datetime.time | ||||
| Return: | ||||
| Boolean : Retorna True si el archivo de datos contiene datos en el rango de | ||||
| fecha especificado, de lo contrario retorna False. | ||||
| Excepciones: | ||||
| Si el archivo no existe o no puede ser abierto | ||||
| Si la cabecera no puede ser leida. | ||||
| """ | ||||
|
Julio Valdez
|
r804 | |||
|
Julio Valdez
|
r514 | try: | ||
|
Julio Valdez
|
r804 | fp = h5py.File(filename,'r') | ||
| grp1 = fp['Data'] | ||||
|
Julio Valdez
|
r514 | except IOError: | ||
|
Julio Valdez
|
r724 | traceback.print_exc() | ||
| raise IOError, "The file %s can't be opened" %(filename) | ||||
|
Julio Valdez
|
r804 | #chino rata | ||
| #In case has utctime attribute | ||||
| grp2 = grp1['utctime'] | ||||
| # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time | ||||
| thisUtcTime = grp2.value[0] | ||||
| fp.close() | ||||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r804 | thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0]) | ||
| thisDate = thisDatetime.date() | ||||
| thisTime = thisDatetime.time() | ||||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r804 | startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds() | ||
| endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds() | ||||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r804 | #General case | ||
| # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o | ||||
| #-----------o----------------------------o----------- | ||||
| # startTime endTime | ||||
| if endTime >= startTime: | ||||
| thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime) | ||||
| if numpy.any(thisUtcLog): #If there is one block between the hours mentioned | ||||
| return thisDatetime | ||||
| return None | ||||
| #If endTime < startTime then endTime belongs to the next day | ||||
| #<<<<<<<<<<<o o>>>>>>>>>>> | ||||
| #-----------o----------------------------o----------- | ||||
| # endTime startTime | ||||
| if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime): | ||||
| return None | ||||
| if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime): | ||||
| return None | ||||
| if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime): | ||||
|
Julio Valdez
|
r514 | return None | ||
| return thisDatetime | ||||
|
Julio Valdez
|
r804 | |||
|
Julio Valdez
|
r514 | def __setNextFileOffline(self): | ||
|
Julio Valdez
|
r804 | self.fileIndex += 1 | ||
| idFile = self.fileIndex | ||||
|
Julio Valdez
|
r514 | if not(idFile < len(self.filenameList)): | ||
| print "No more Files" | ||||
| return 0 | ||||
| filename = self.filenameList[idFile] | ||||
| filePointer = h5py.File(filename,'r') | ||||
|
Julio Valdez
|
r804 | |||
|
Julio Valdez
|
r514 | self.filename = filename | ||
| self.fp = filePointer | ||||
| print "Setting the file: %s"%self.filename | ||||
|
Julio Valdez
|
r804 | # self.__readMetadata() | ||
|
Julio Valdez
|
r543 | self.__setBlockList() | ||
|
Julio Valdez
|
r804 | self.__readData() | ||
|
Julio Valdez
|
r543 | # self.nRecords = self.fp['Data'].attrs['blocksPerFile'] | ||
|
Julio Valdez
|
r804 | # self.nRecords = self.fp['Data'].attrs['nRecords'] | ||
|
Julio Valdez
|
r543 | self.blockIndex = 0 | ||
|
Julio Valdez
|
r514 | return 1 | ||
|
Julio Valdez
|
r543 | def __setBlockList(self): | ||
| ''' | ||||
|
Julio Valdez
|
r804 | Selects the data within the times defined | ||
|
Julio Valdez
|
r543 | self.fp | ||
|
Julio Valdez
|
r804 | self.startTime | ||
| self.endTime | ||||
|
Julio Valdez
|
r543 | |||
| self.blockList | ||||
| self.blocksPerFile | ||||
| ''' | ||||
|
Julio Valdez
|
r804 | fp = self.fp | ||
| startTime = self.startTime | ||||
| endTime = self.endTime | ||||
|
Julio Valdez
|
r543 | |||
|
Julio Valdez
|
r804 | grp = fp['Data'] | ||
| thisUtcTime = grp['utctime'].value.astype(numpy.float)[0] | ||||
|
Julio Valdez
|
r543 | |||
| if self.timezone == 'lt': | ||||
|
Julio Valdez
|
r804 | thisUtcTime -= 5*3600 | ||
| thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0]) | ||||
| thisDate = thisDatetime.date() | ||||
| thisTime = thisDatetime.time() | ||||
|
Julio Valdez
|
r543 | |||
|
Julio Valdez
|
r804 | startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds() | ||
| endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds() | ||||
| ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0] | ||||
|
Julio Valdez
|
r543 | |||
| self.blockList = ind | ||||
| self.blocksPerFile = len(ind) | ||||
| return | ||||
|
Julio Valdez
|
r514 | def __readMetadata(self): | ||
|
Julio Valdez
|
r543 | ''' | ||
|
Julio Valdez
|
r804 | Reads Metadata | ||
|
Julio Valdez
|
r543 | self.pathMeta | ||
| self.listShapes | ||||
| self.listMetaname | ||||
| self.listMeta | ||||
| ''' | ||||
|
Julio Valdez
|
r804 | # grp = self.fp['Data'] | ||
| # pathMeta = os.path.join(self.path, grp.attrs['metadata']) | ||||
| # | ||||
| # if pathMeta == self.pathMeta: | ||||
| # return | ||||
| # else: | ||||
| # self.pathMeta = pathMeta | ||||
| # | ||||
| # filePointer = h5py.File(self.pathMeta,'r') | ||||
| # groupPointer = filePointer['Metadata'] | ||||
| filename = self.filenameList[0] | ||||
| fp = h5py.File(filename,'r') | ||||
| gp = fp['Metadata'] | ||||
|
Julio Valdez
|
r514 | |||
| listMetaname = [] | ||||
| listMetadata = [] | ||||
|
Julio Valdez
|
r804 | for item in gp.items(): | ||
|
Julio Valdez
|
r514 | name = item[0] | ||
|
Julio Valdez
|
r543 | if name=='array dimensions': | ||
|
Julio Valdez
|
r804 | table = gp[name][:] | ||
|
Julio Valdez
|
r543 | listShapes = {} | ||
| for shapes in table: | ||||
|
Julio Valdez
|
r804 | listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]]) | ||
|
Julio Valdez
|
r514 | else: | ||
|
Julio Valdez
|
r804 | data = gp[name].value | ||
|
Julio Valdez
|
r514 | listMetaname.append(name) | ||
| listMetadata.append(data) | ||||
|
Julio Valdez
|
r804 | # if name=='type': | ||
| # self.__initDataOut(data) | ||||
|
Julio Valdez
|
r543 | self.listShapes = listShapes | ||
| self.listMetaname = listMetaname | ||||
| self.listMeta = listMetadata | ||||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r804 | fp.close() | ||
|
Julio Valdez
|
r514 | return | ||
|
Julio Valdez
|
r543 | def __readData(self): | ||
| grp = self.fp['Data'] | ||||
| listdataname = [] | ||||
| listdata = [] | ||||
| for item in grp.items(): | ||||
| name = item[0] | ||||
| listdataname.append(name) | ||||
|
Julio Valdez
|
r804 | |||
| array = self.__setDataArray(grp[name],self.listShapes[name]) | ||||
|
Julio Valdez
|
r543 | listdata.append(array) | ||
| self.listDataname = listdataname | ||||
| self.listData = listdata | ||||
| return | ||||
|
Julio Valdez
|
r804 | def __setDataArray(self, dataset, shapes): | ||
| nDims = shapes[0] | ||||
|
Julio Valdez
|
r543 | |||
|
Julio Valdez
|
r804 | nDim2 = shapes[1] #Dimension 0 | ||
|
Julio Valdez
|
r543 | |||
|
Julio Valdez
|
r804 | nDim1 = shapes[2] #Dimension 1, number of Points or Parameters | ||
|
Julio Valdez
|
r543 | |||
|
Julio Valdez
|
r804 | nDim0 = shapes[3] #Dimension 2, number of samples or ranges | ||
|
Julio Valdez
|
r543 | |||
|
Julio Valdez
|
r804 | mode = shapes[4] #Mode of storing | ||
|
Julio Valdez
|
r543 | |||
|
Julio Valdez
|
r804 | blockList = self.blockList | ||
| blocksPerFile = self.blocksPerFile | ||||
| #Depending on what mode the data was stored | ||||
| # if mode == 0: #Divided in channels | ||||
| # strds = 'channel' | ||||
| # nDatas = nDim2 | ||||
| # newShapes = (blocksPerFile,nDim1,nDim0) | ||||
| if mode == 1: #Divided in parameter | ||||
|
Julio Valdez
|
r543 | strds = 'param' | ||
|
Julio Valdez
|
r804 | nDatas = nDim1 | ||
| newShapes = (blocksPerFile,nDim2,nDim0) | ||||
| elif mode==2: #Concatenated in a table | ||||
| strds = 'table0' | ||||
| arrayData = dataset[strds].value | ||||
| #Selecting part of the dataset | ||||
| utctime = arrayData[:,0] | ||||
| u, indices = numpy.unique(utctime, return_index=True) | ||||
|
Julio Valdez
|
r543 | |||
|
Julio Valdez
|
r804 | if blockList.size != indices.size: | ||
| indMin = indices[blockList[0]] | ||||
| indMax = indices[blockList[-1] + 1] | ||||
| arrayData = arrayData[indMin:indMax,:] | ||||
| return arrayData | ||||
| #------- One dimension --------------- | ||||
| if nDims == 1: | ||||
| arrayData = dataset.value.astype(numpy.float)[0][blockList] | ||||
| #------- Two dimensions ----------- | ||||
| elif nDims == 2: | ||||
| arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0)) | ||||
| newShapes = (blocksPerFile,nDim0) | ||||
| nDatas = nDim1 | ||||
| for i in range(nDatas): | ||||
| data = dataset[strds + str(i)].value | ||||
| arrayData[:,i,:] = data[blockList,:] | ||||
| #------- Three dimensions --------- | ||||
| else: | ||||
| arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0)) | ||||
| for i in range(nDatas): | ||||
| data = dataset[strds + str(i)].value | ||||
| data = data[blockList,:,:] | ||||
| data = data.reshape(newShapes) | ||||
| # if mode == 0: | ||||
| # arrayData[:,i,:,:] = data | ||||
| # else: | ||||
|
Julio Valdez
|
r543 | arrayData[:,:,i,:] = data | ||
|
Julio Valdez
|
r804 | |||
|
Julio Valdez
|
r543 | return arrayData | ||
|
Julio Valdez
|
r514 | |||
| def __setDataOut(self): | ||||
|
Julio Valdez
|
r543 | listMeta = self.listMeta | ||
|
Julio Valdez
|
r514 | listMetaname = self.listMetaname | ||
| listDataname = self.listDataname | ||||
| listData = self.listData | ||||
|
Julio Valdez
|
r804 | listShapes = self.listShapes | ||
|
Julio Valdez
|
r514 | |||
| blockIndex = self.blockIndex | ||||
|
Julio Valdez
|
r804 | # blockList = self.blockList | ||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r543 | for i in range(len(listMeta)): | ||
| setattr(self.dataOut,listMetaname[i],listMeta[i]) | ||||
|
Julio Valdez
|
r514 | |||
| for j in range(len(listData)): | ||||
|
Julio Valdez
|
r804 | nShapes = listShapes[listDataname[j]][0] | ||
| mode = listShapes[listDataname[j]][4] | ||||
| if nShapes == 1: | ||||
| setattr(self.dataOut,listDataname[j],listData[j][blockIndex]) | ||||
| elif nShapes > 1: | ||||
| setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:]) | ||||
| #Mode Meteors | ||||
| elif mode ==2: | ||||
| selectedData = self.__selectDataMode2(listData[j], blockIndex) | ||||
| setattr(self.dataOut, listDataname[j], selectedData) | ||||
| return | ||||
| def __selectDataMode2(self, data, blockIndex): | ||||
| utctime = data[:,0] | ||||
| aux, indices = numpy.unique(utctime, return_inverse=True) | ||||
| selInd = numpy.where(indices == blockIndex)[0] | ||||
| selData = data[selInd,:] | ||||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r804 | return selData | ||
|
Julio Valdez
|
r514 | |||
| def getData(self): | ||||
|
Julio Valdez
|
r543 | # if self.flagNoMoreFiles: | ||
| # self.dataOut.flagNoData = True | ||||
| # print 'Process finished' | ||||
| # return 0 | ||||
| # | ||||
| if self.blockIndex==self.blocksPerFile: | ||||
| if not( self.__setNextFileOffline() ): | ||||
| self.dataOut.flagNoData = True | ||||
| return 0 | ||||
|
Julio Valdez
|
r804 | |||
|
Julio Valdez
|
r543 | # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers | ||
| # self.dataOut.flagNoData = True | ||||
| # return 0 | ||||
|
Julio Valdez
|
r804 | # self.__readData() | ||
|
Julio Valdez
|
r514 | self.__setDataOut() | ||
| self.dataOut.flagNoData = False | ||||
| self.blockIndex += 1 | ||||
|
Julio Valdez
|
r543 | return | ||
|
Julio Valdez
|
r514 | |||
| def run(self, **kwargs): | ||||
| if not(self.isConfig): | ||||
| self.setup(**kwargs) | ||||
|
Julio Valdez
|
r543 | # self.setObjProperties() | ||
|
Julio Valdez
|
r514 | self.isConfig = True | ||
| self.getData() | ||||
| return | ||||
| class HDF5Writer(Operation): | ||||
|
Julio Valdez
|
r804 | ''' | ||
| HDF5 Writer, stores parameters data in HDF5 format files | ||||
| path: path where the files will be stored | ||||
| blocksPerFile: number of blocks that will be saved in per HDF5 format file | ||||
| mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors) | ||||
| metadataList: list of attributes that will be stored as metadata | ||||
| dataList: list of attributes that will be stores as data | ||||
| ''' | ||||
|
Julio Valdez
|
r514 | |||
| ext = ".hdf5" | ||||
| optchar = "D" | ||||
| metaoptchar = "M" | ||||
| metaFile = None | ||||
|
Julio Valdez
|
r608 | filename = None | ||
|
Julio Valdez
|
r514 | path = None | ||
| setFile = None | ||||
| fp = None | ||||
| grp = None | ||||
| ds = None | ||||
| firsttime = True | ||||
| #Configurations | ||||
| blocksPerFile = None | ||||
| blockIndex = None | ||||
| dataOut = None | ||||
| #Data Arrays | ||||
| dataList = None | ||||
| metadataList = None | ||||
|
Julio Valdez
|
r810 | # arrayDim = None | ||
| dsList = None #List of dictionaries | ||||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r515 | tableDim = None | ||
|
Julio Valdez
|
r543 | # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')] | ||
| dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')] | ||||
| mode = None | ||||
| nDatas = None #Number of datasets to be stored per array | ||||
|
Julio Valdez
|
r608 | |||
|
Julio Valdez
|
r543 | nDims = None #Number Dimensions in each dataset | ||
|
Julio Valdez
|
r515 | |||
|
Julio Valdez
|
r608 | nDimsForDs = None | ||
|
Julio Valdez
|
r724 | currentDay = None | ||
|
Julio Valdez
|
r514 | def __init__(self): | ||
| Operation.__init__(self) | ||||
| self.isConfig = False | ||||
| return | ||||
| def setup(self, dataOut, **kwargs): | ||||
| self.path = kwargs['path'] | ||||
| if kwargs.has_key('blocksPerFile'): | ||||
| self.blocksPerFile = kwargs['blocksPerFile'] | ||||
| else: | ||||
| self.blocksPerFile = 10 | ||||
|
Julio Valdez
|
r543 | self.metadataList = kwargs['metadataList'] | ||
| self.dataList = kwargs['dataList'] | ||||
|
Julio Valdez
|
r514 | self.dataOut = dataOut | ||
|
Julio Valdez
|
r543 | if kwargs.has_key('mode'): | ||
| mode = kwargs['mode'] | ||||
| if type(mode) == int: | ||||
| mode = numpy.zeros(len(self.dataList)) + mode | ||||
| else: | ||||
|
Julio Valdez
|
r804 | mode = numpy.ones(len(self.dataList)) | ||
|
Julio Valdez
|
r543 | |||
| self.mode = mode | ||||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r543 | arrayDim = numpy.zeros((len(self.dataList),5)) | ||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r543 | #Table dimensions | ||
|
Julio Valdez
|
r515 | dtype0 = self.dtype | ||
| tableList = [] | ||||
|
Julio Valdez
|
r810 | #Dictionary and list of tables | ||
| dsList = [] | ||||
|
Julio Valdez
|
r514 | for i in range(len(self.dataList)): | ||
|
Julio Valdez
|
r810 | dsDict = {} | ||
|
Julio Valdez
|
r543 | dataAux = getattr(self.dataOut, self.dataList[i]) | ||
|
Julio Valdez
|
r810 | dsDict['variable'] = self.dataList[i] | ||
|
Julio Valdez
|
r804 | #--------------------- Conditionals ------------------------ | ||
| #There is no data | ||||
| if dataAux == None: | ||||
| return 0 | ||||
| #Not array, just a number | ||||
|
Julio Valdez
|
r810 | #Mode 0 | ||
| if type(dataAux)==float or type(dataAux)==int: | ||||
| dsDict['mode'] = 0 | ||||
| dsDict['nDim'] = 0 | ||||
|
Julio Valdez
|
r543 | arrayDim[i,0] = 1 | ||
|
Julio Valdez
|
r810 | dsList.append(dsDict) | ||
| #Mode 2: meteors | ||||
| elif mode[i] == 2: | ||||
| # dsDict['nDim'] = 0 | ||||
| dsDict['dsName'] = 'table0' | ||||
| dsDict['mode'] = 2 # Mode meteors | ||||
| dsDict['shape'] = dataAux.shape[-1] | ||||
| dsDict['nDim'] = 0 | ||||
|
Julio Valdez
|
r804 | arrayDim[i,3] = dataAux.shape[-1] | ||
| arrayDim[i,4] = mode[i] #Mode the data was stored | ||||
|
Julio Valdez
|
r810 | |||
| dsList.append(dsDict) | ||||
|
Julio Valdez
|
r804 | |||
|
Julio Valdez
|
r810 | #Mode 1 | ||
|
Julio Valdez
|
r514 | else: | ||
|
Julio Valdez
|
r804 | arrayDim0 = dataAux.shape #Data dimensions | ||
| arrayDim[i,0] = len(arrayDim0) #Number of array dimensions | ||||
| arrayDim[i,4] = mode[i] #Mode the data was stored | ||||
|
Julio Valdez
|
r724 | |||
|
Julio Valdez
|
r810 | strtable = 'table' | ||
| dsDict['mode'] = 1 # Mode parameters | ||||
|
Julio Valdez
|
r804 | # Three-dimension arrays | ||
|
Julio Valdez
|
r543 | if len(arrayDim0) == 3: | ||
| arrayDim[i,1:-1] = numpy.array(arrayDim0) | ||||
|
Julio Valdez
|
r810 | nTables = int(arrayDim[i,2]) | ||
| dsDict['dsNumber'] = nTables | ||||
| dsDict['shape'] = arrayDim[i,2:4] | ||||
| dsDict['nDim'] = 3 | ||||
| for j in range(nTables): | ||||
| dsDict = dsDict.copy() | ||||
| dsDict['dsName'] = strtable + str(j) | ||||
| dsList.append(dsDict) | ||||
|
Julio Valdez
|
r804 | # Two-dimension arrays | ||
|
Julio Valdez
|
r543 | elif len(arrayDim0) == 2: | ||
|
Julio Valdez
|
r804 | arrayDim[i,2:-1] = numpy.array(arrayDim0) | ||
|
Julio Valdez
|
r810 | nTables = int(arrayDim[i,2]) | ||
| dsDict['dsNumber'] = nTables | ||||
| dsDict['shape'] = arrayDim[i,3] | ||||
| dsDict['nDim'] = 2 | ||||
| for j in range(nTables): | ||||
| dsDict = dsDict.copy() | ||||
| dsDict['dsName'] = strtable + str(j) | ||||
| dsList.append(dsDict) | ||||
|
Julio Valdez
|
r804 | # One-dimension arrays | ||
|
Julio Valdez
|
r543 | elif len(arrayDim0) == 1: | ||
|
Julio Valdez
|
r810 | arrayDim[i,3] = arrayDim0[0] | ||
| dsDict['shape'] = arrayDim0[0] | ||||
| dsDict['dsNumber'] = 1 | ||||
| dsDict['dsName'] = strtable + str(0) | ||||
| dsDict['nDim'] = 1 | ||||
| dsList.append(dsDict) | ||||
|
Julio Valdez
|
r543 | |||
| table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0) | ||||
|
Julio Valdez
|
r515 | tableList.append(table) | ||
|
Julio Valdez
|
r543 | |||
|
Julio Valdez
|
r810 | # self.arrayDim = arrayDim | ||
| self.dsList = dsList | ||||
|
Julio Valdez
|
r515 | self.tableDim = numpy.array(tableList, dtype = dtype0) | ||
|
Julio Valdez
|
r514 | self.blockIndex = 0 | ||
|
Julio Valdez
|
r724 | timeTuple = time.localtime(dataOut.utctime) | ||
| self.currentDay = timeTuple.tm_yday | ||||
| return 1 | ||||
|
Julio Valdez
|
r514 | |||
| def putMetadata(self): | ||||
| fp = self.createMetadataFile() | ||||
| self.writeMetadata(fp) | ||||
| fp.close() | ||||
| return | ||||
| def createMetadataFile(self): | ||||
| ext = self.ext | ||||
| path = self.path | ||||
| setFile = self.setFile | ||||
| timeTuple = time.localtime(self.dataOut.utctime) | ||||
|
Julio Valdez
|
r608 | |||
| subfolder = '' | ||||
|
Julio Valdez
|
r514 | fullpath = os.path.join( path, subfolder ) | ||
|
Julio Valdez
|
r608 | |||
|
Julio Valdez
|
r514 | if not( os.path.exists(fullpath) ): | ||
| os.mkdir(fullpath) | ||||
| setFile = -1 #inicializo mi contador de seteo | ||||
|
Julio Valdez
|
r608 | |||
| subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) | ||||
| fullpath = os.path.join( path, subfolder ) | ||||
| if not( os.path.exists(fullpath) ): | ||||
| os.mkdir(fullpath) | ||||
| setFile = -1 #inicializo mi contador de seteo | ||||
|
Julio Valdez
|
r514 | else: | ||
| filesList = os.listdir( fullpath ) | ||||
|
Julio Valdez
|
r608 | filesList = sorted( filesList, key=str.lower ) | ||
|
Julio Valdez
|
r514 | if len( filesList ) > 0: | ||
|
Julio Valdez
|
r608 | filesList = [k for k in filesList if 'M' in k] | ||
|
Julio Valdez
|
r514 | filen = filesList[-1] | ||
| # el filename debera tener el siguiente formato | ||||
| # 0 1234 567 89A BCDE (hex) | ||||
| # x YYYY DDD SSS .ext | ||||
| if isNumber( filen[8:11] ): | ||||
| setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file | ||||
| else: | ||||
| setFile = -1 | ||||
| else: | ||||
| setFile = -1 #inicializo mi contador de seteo | ||||
| setFile += 1 | ||||
|
Julio Valdez
|
r608 | |||
|
Julio Valdez
|
r514 | file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar, | ||
| timeTuple.tm_year, | ||||
| timeTuple.tm_yday, | ||||
| setFile, | ||||
| ext ) | ||||
| filename = os.path.join( path, subfolder, file ) | ||||
| self.metaFile = file | ||||
| #Setting HDF5 File | ||||
| fp = h5py.File(filename,'w') | ||||
| return fp | ||||
| def writeMetadata(self, fp): | ||||
| grp = fp.create_group("Metadata") | ||||
|
Julio Valdez
|
r515 | grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype) | ||
|
Julio Valdez
|
r514 | |||
| for i in range(len(self.metadataList)): | ||||
| grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i])) | ||||
| return | ||||
|
Julio Valdez
|
r724 | def dateFlag(self): | ||
| timeTuple = time.localtime(self.dataOut.utctime) | ||||
| dataDay = timeTuple.tm_yday | ||||
| if dataDay == self.currentDay: | ||||
| return False | ||||
| self.currentDay = dataDay | ||||
| return True | ||||
|
Julio Valdez
|
r514 | def setNextFile(self): | ||
| ext = self.ext | ||||
| path = self.path | ||||
| setFile = self.setFile | ||||
|
Julio Valdez
|
r543 | mode = self.mode | ||
|
Julio Valdez
|
r608 | |||
|
Julio Valdez
|
r514 | timeTuple = time.localtime(self.dataOut.utctime) | ||
| subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) | ||||
| fullpath = os.path.join( path, subfolder ) | ||||
|
Julio Valdez
|
r608 | |||
| if os.path.exists(fullpath): | ||||
|
Julio Valdez
|
r514 | filesList = os.listdir( fullpath ) | ||
|
Julio Valdez
|
r608 | filesList = [k for k in filesList if 'D' in k] | ||
|
Julio Valdez
|
r514 | if len( filesList ) > 0: | ||
| filesList = sorted( filesList, key=str.lower ) | ||||
| filen = filesList[-1] | ||||
| # el filename debera tener el siguiente formato | ||||
| # 0 1234 567 89A BCDE (hex) | ||||
| # x YYYY DDD SSS .ext | ||||
| if isNumber( filen[8:11] ): | ||||
| setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file | ||||
| else: | ||||
| setFile = -1 | ||||
| else: | ||||
| setFile = -1 #inicializo mi contador de seteo | ||||
|
Julio Valdez
|
r724 | else: | ||
| os.mkdir(fullpath) | ||||
| setFile = -1 #inicializo mi contador de seteo | ||||
|
Julio Valdez
|
r514 | setFile += 1 | ||
| file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, | ||||
| timeTuple.tm_year, | ||||
| timeTuple.tm_yday, | ||||
| setFile, | ||||
| ext ) | ||||
| filename = os.path.join( path, subfolder, file ) | ||||
|
Julio Valdez
|
r810 | #Setting HDF5 File | ||
| fp = h5py.File(filename,'w') | ||||
| #write metadata | ||||
| self.writeMetadata(fp) | ||||
| #Write data | ||||
| grp = fp.create_group("Data") | ||||
| # grp.attrs['metadata'] = self.metaFile | ||||
| # grp.attrs['blocksPerFile'] = 0 | ||||
| ds = [] | ||||
| data = [] | ||||
| dsList = self.dsList | ||||
| i = 0 | ||||
| while i < len(dsList): | ||||
| dsInfo = dsList[i] | ||||
| #One-dimension data | ||||
| if dsInfo['mode'] == 0: | ||||
| # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20') | ||||
| ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64) | ||||
| ds.append(ds0) | ||||
| data.append([]) | ||||
| i += 1 | ||||
| continue | ||||
| # nDimsForDs.append(nDims[i]) | ||||
| elif dsInfo['mode'] == 2: | ||||
| grp0 = grp.create_group(dsInfo['variable']) | ||||
| ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True) | ||||
| ds.append(ds0) | ||||
| data.append([]) | ||||
| i += 1 | ||||
| continue | ||||
| elif dsInfo['mode'] == 1: | ||||
| grp0 = grp.create_group(dsInfo['variable']) | ||||
| for j in range(dsInfo['dsNumber']): | ||||
| dsInfo = dsList[i] | ||||
| tableName = dsInfo['dsName'] | ||||
| shape = dsInfo['shape'] | ||||
| if dsInfo['nDim'] == 3: | ||||
| ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True) | ||||
| else: | ||||
| ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True) | ||||
| ds.append(ds0) | ||||
| data.append([]) | ||||
| i += 1 | ||||
| # nDimsForDs.append(nDims[i]) | ||||
| fp.flush() | ||||
| fp.close() | ||||
| # self.nDatas = nDatas | ||||
| # self.nDims = nDims | ||||
| # self.nDimsForDs = nDimsForDs | ||||
| #Saving variables | ||||
| print 'Writing the file: %s'%filename | ||||
| self.filename = filename | ||||
| # self.fp = fp | ||||
| # self.grp = grp | ||||
| # self.grp.attrs.modify('nRecords', 1) | ||||
| self.ds = ds | ||||
| self.data = data | ||||
| # self.setFile = setFile | ||||
| self.firsttime = True | ||||
| self.blockIndex = 0 | ||||
| return | ||||
| def setNextFile1(self): | ||||
| ext = self.ext | ||||
| path = self.path | ||||
| setFile = self.setFile | ||||
| mode = self.mode | ||||
| timeTuple = time.localtime(self.dataOut.utctime) | ||||
| subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) | ||||
| fullpath = os.path.join( path, subfolder ) | ||||
| if os.path.exists(fullpath): | ||||
| filesList = os.listdir( fullpath ) | ||||
| filesList = [k for k in filesList if 'D' in k] | ||||
| if len( filesList ) > 0: | ||||
| filesList = sorted( filesList, key=str.lower ) | ||||
| filen = filesList[-1] | ||||
| # el filename debera tener el siguiente formato | ||||
| # 0 1234 567 89A BCDE (hex) | ||||
| # x YYYY DDD SSS .ext | ||||
| if isNumber( filen[8:11] ): | ||||
| setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file | ||||
| else: | ||||
| setFile = -1 | ||||
| else: | ||||
| setFile = -1 #inicializo mi contador de seteo | ||||
| else: | ||||
| os.mkdir(fullpath) | ||||
| setFile = -1 #inicializo mi contador de seteo | ||||
| setFile += 1 | ||||
| file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, | ||||
| timeTuple.tm_year, | ||||
| timeTuple.tm_yday, | ||||
| setFile, | ||||
| ext ) | ||||
| filename = os.path.join( path, subfolder, file ) | ||||
|
Julio Valdez
|
r514 | #Setting HDF5 File | ||
| fp = h5py.File(filename,'w') | ||||
|
Julio Valdez
|
r724 | |||
| #writemetadata | ||||
| self.writeMetadata(fp) | ||||
|
Julio Valdez
|
r514 | grp = fp.create_group("Data") | ||
|
Julio Valdez
|
r724 | # grp.attrs['metadata'] = self.metaFile | ||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r543 | # grp.attrs['blocksPerFile'] = 0 | ||
|
Julio Valdez
|
r514 | |||
| ds = [] | ||||
| data = [] | ||||
|
Julio Valdez
|
r608 | nDimsForDs = [] | ||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r543 | nDatas = numpy.zeros(len(self.dataList)) | ||
| nDims = self.arrayDim[:,0] | ||||
|
Julio Valdez
|
r608 | nDim1 = self.arrayDim[:,2] | ||
| nDim0 = self.arrayDim[:,3] | ||||
|
Julio Valdez
|
r543 | for i in range(len(self.dataList)): | ||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r804 | #One-dimension data | ||
|
Julio Valdez
|
r543 | if nDims[i]==1: | ||
|
Julio Valdez
|
r608 | # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20') | ||
| ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64) | ||||
|
Julio Valdez
|
r514 | ds.append(ds0) | ||
| data.append([]) | ||||
|
Julio Valdez
|
r608 | nDimsForDs.append(nDims[i]) | ||
|
Julio Valdez
|
r543 | else: | ||
|
Julio Valdez
|
r804 | #Channel mode | ||
| # if mode[i] == 0: | ||||
| # strMode = "channel" | ||||
| # | ||||
| # #nDatas is the number of arrays per variable | ||||
| # if nDims[i] == 1: | ||||
| # nDatas[i] = self.arrayDim[i,1] | ||||
| # elif nDims[i] == 2: | ||||
| # nDatas[i] = self.arrayDim[i,2] | ||||
| #Parameters mode | ||||
| if mode[i] == 1: | ||||
|
Julio Valdez
|
r543 | strMode = "param" | ||
| nDatas[i] = self.arrayDim[i,2] | ||||
|
Julio Valdez
|
r804 | |||
| #Meteors mode | ||||
| elif mode[i] == 2: | ||||
| strMode = "table" | ||||
| nDatas[i] = 1 | ||||
|
Julio Valdez
|
r543 | grp0 = grp.create_group(self.dataList[i]) | ||
| for j in range(int(nDatas[i])): | ||||
| tableName = strMode + str(j) | ||||
| if nDims[i] == 3: | ||||
|
Julio Valdez
|
r608 | ds0 = grp0.create_dataset(tableName, (nDim1[i],nDim0[i],1) , data = numpy.zeros((nDim1[i],nDim0[i],1)) ,maxshape=(None,nDim0[i],None), chunks=True) | ||
|
Julio Valdez
|
r804 | |||
|
Julio Valdez
|
r543 | else: | ||
|
Julio Valdez
|
r608 | ds0 = grp0.create_dataset(tableName, (1,nDim0[i]), data = numpy.zeros((1,nDim0[i])) , maxshape=(None,nDim0[i]), chunks=True) | ||
|
Julio Valdez
|
r543 | |||
| ds.append(ds0) | ||||
| data.append([]) | ||||
|
Julio Valdez
|
r608 | nDimsForDs.append(nDims[i]) | ||
|
Julio Valdez
|
r804 | |||
| fp.flush() | ||||
| fp.close() | ||||
|
Julio Valdez
|
r543 | self.nDatas = nDatas | ||
| self.nDims = nDims | ||||
|
Julio Valdez
|
r608 | self.nDimsForDs = nDimsForDs | ||
|
Julio Valdez
|
r514 | #Saving variables | ||
| print 'Writing the file: %s'%filename | ||||
|
Julio Valdez
|
r608 | self.filename = filename | ||
|
Julio Valdez
|
r804 | # self.fp = fp | ||
| # self.grp = grp | ||||
| # self.grp.attrs.modify('nRecords', 1) | ||||
|
Julio Valdez
|
r514 | self.ds = ds | ||
| self.data = data | ||||
|
Julio Valdez
|
r804 | # | ||
| # self.setFile = setFile | ||||
|
Julio Valdez
|
r514 | self.firsttime = True | ||
| self.blockIndex = 0 | ||||
| return | ||||
| def putData(self): | ||||
|
Julio Valdez
|
r608 | |||
|
Julio Valdez
|
r724 | if self.blockIndex == self.blocksPerFile or self.dateFlag(): | ||
|
Julio Valdez
|
r608 | self.setNextFile() | ||
|
Julio Valdez
|
r804 | # if not self.firsttime: | ||
| self.readBlock() | ||||
| self.setBlock() #Prepare data to be written | ||||
| self.writeBlock() #Write data | ||||
|
Julio Valdez
|
r608 | |||
|
Julio Valdez
|
r514 | return | ||
|
Julio Valdez
|
r608 | def readBlock(self): | ||
| ''' | ||||
| data Array configured | ||||
| self.data | ||||
| ''' | ||||
|
Julio Valdez
|
r810 | dsList = self.dsList | ||
|
Julio Valdez
|
r608 | ds = self.ds | ||
| #Setting HDF5 File | ||||
| fp = h5py.File(self.filename,'r+') | ||||
| grp = fp["Data"] | ||||
| ind = 0 | ||||
| # grp.attrs['blocksPerFile'] = 0 | ||||
|
Julio Valdez
|
r810 | while ind < len(dsList): | ||
| dsInfo = dsList[ind] | ||||
|
Julio Valdez
|
r608 | |||
|
Julio Valdez
|
r810 | if dsInfo['mode'] == 0: | ||
| ds0 = grp[dsInfo['variable']] | ||||
|
Julio Valdez
|
r608 | ds[ind] = ds0 | ||
| ind += 1 | ||||
| else: | ||||
|
Julio Valdez
|
r804 | |||
|
Julio Valdez
|
r810 | grp0 = grp[dsInfo['variable']] | ||
|
Julio Valdez
|
r608 | |||
|
Julio Valdez
|
r810 | for j in range(dsInfo['dsNumber']): | ||
| dsInfo = dsList[ind] | ||||
| ds0 = grp0[dsInfo['dsName']] | ||||
|
Julio Valdez
|
r608 | ds[ind] = ds0 | ||
| ind += 1 | ||||
|
Julio Valdez
|
r804 | |||
|
Julio Valdez
|
r608 | self.fp = fp | ||
| self.grp = grp | ||||
| self.ds = ds | ||||
| return | ||||
|
Julio Valdez
|
r514 | def setBlock(self): | ||
|
Julio Valdez
|
r515 | ''' | ||
| data Array configured | ||||
|
Julio Valdez
|
r543 | |||
| self.data | ||||
|
Julio Valdez
|
r515 | ''' | ||
|
Julio Valdez
|
r514 | #Creating Arrays | ||
|
Julio Valdez
|
r810 | dsList = self.dsList | ||
|
Julio Valdez
|
r514 | data = self.data | ||
| ind = 0 | ||||
|
Julio Valdez
|
r543 | |||
|
Julio Valdez
|
r810 | while ind < len(dsList): | ||
| dsInfo = dsList[ind] | ||||
| dataAux = getattr(self.dataOut, dsInfo['variable']) | ||||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r810 | mode = dsInfo['mode'] | ||
| nDim = dsInfo['nDim'] | ||||
| if mode == 0 or mode == 2 or nDim == 1: | ||||
|
Julio Valdez
|
r608 | data[ind] = dataAux | ||
|
Julio Valdez
|
r810 | ind += 1 | ||
| # elif nDim == 1: | ||||
| # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1)) | ||||
| # ind += 1 | ||||
| elif nDim == 2: | ||||
| for j in range(dsInfo['dsNumber']): | ||||
|
Julio Valdez
|
r804 | data[ind] = dataAux[j,:] | ||
| ind += 1 | ||||
|
Julio Valdez
|
r810 | elif nDim == 3: | ||
| for j in range(dsInfo['dsNumber']): | ||||
|
Julio Valdez
|
r804 | data[ind] = dataAux[:,j,:] | ||
|
Julio Valdez
|
r543 | ind += 1 | ||
|
Julio Valdez
|
r810 | |||
|
Julio Valdez
|
r514 | self.data = data | ||
| return | ||||
| def writeBlock(self): | ||||
|
Julio Valdez
|
r515 | ''' | ||
| Saves the block in the HDF5 file | ||||
| ''' | ||||
|
Julio Valdez
|
r810 | dsList = self.dsList | ||
|
Julio Valdez
|
r514 | for i in range(len(self.ds)): | ||
|
Julio Valdez
|
r810 | dsInfo = dsList[i] | ||
| nDim = dsInfo['nDim'] | ||||
| mode = dsInfo['mode'] | ||||
|
Julio Valdez
|
r804 | |||
| # First time | ||||
|
Julio Valdez
|
r608 | if self.firsttime: | ||
| # self.ds[i].resize(self.data[i].shape) | ||||
| # self.ds[i][self.blockIndex,:] = self.data[i] | ||||
| if type(self.data[i]) == numpy.ndarray: | ||||
|
Julio Valdez
|
r810 | if nDim == 3: | ||
|
Julio Valdez
|
r608 | self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1)) | ||
|
Julio Valdez
|
r810 | self.ds[i].resize(self.data[i].shape) | ||
|
Julio Valdez
|
r804 | |||
| self.ds[i][:] = self.data[i] | ||||
|
Julio Valdez
|
r608 | else: | ||
|
Julio Valdez
|
r804 | |||
| # From second time | ||||
| # Meteors! | ||||
|
Julio Valdez
|
r810 | if mode == 2: | ||
|
Julio Valdez
|
r804 | dataShape = self.data[i].shape | ||
| dsShape = self.ds[i].shape | ||||
| self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1])) | ||||
| self.ds[i][dsShape[0]:,:] = self.data[i] | ||||
|
Julio Valdez
|
r810 | # No dimension | ||
| elif mode == 0: | ||||
|
Julio Valdez
|
r608 | self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1)) | ||
| self.ds[i][0,-1] = self.data[i] | ||||
|
Julio Valdez
|
r810 | # One dimension | ||
| elif nDim == 1: | ||||
| self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1])) | ||||
| self.ds[i][-1,:] = self.data[i] | ||||
|
Julio Valdez
|
r804 | # Two dimension | ||
|
Julio Valdez
|
r810 | elif nDim == 2: | ||
|
Julio Valdez
|
r608 | self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1])) | ||
| self.ds[i][self.blockIndex,:] = self.data[i] | ||||
|
Julio Valdez
|
r804 | # Three dimensions | ||
|
Julio Valdez
|
r810 | elif nDim == 3: | ||
|
Julio Valdez
|
r804 | self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1)) | ||
| self.ds[i][:,:,-1] = self.data[i] | ||||
|
Julio Valdez
|
r514 | |||
|
Julio Valdez
|
r804 | self.firsttime = False | ||
|
Julio Valdez
|
r514 | self.blockIndex += 1 | ||
|
Julio Valdez
|
r804 | |||
| #Close to save changes | ||||
| self.fp.flush() | ||||
| self.fp.close() | ||||
|
Julio Valdez
|
r514 | return | ||
| def run(self, dataOut, **kwargs): | ||||
|
Julio Valdez
|
r724 | |||
|
Julio Valdez
|
r514 | if not(self.isConfig): | ||
|
Julio Valdez
|
r724 | flagdata = self.setup(dataOut, **kwargs) | ||
| if not(flagdata): | ||||
| return | ||||
|
Julio Valdez
|
r514 | self.isConfig = True | ||
|
Julio Valdez
|
r724 | # self.putMetadata() | ||
|
Julio Valdez
|
r514 | self.setNextFile() | ||
| self.putData() | ||||
| return | ||||
|
Julio Valdez
|
r724 | |||