jroIO_param.py
1094 lines
| 33.1 KiB
| text/x-python
|
PythonLexer
Julio Valdez
|
r848 | import numpy | |
| import time | |||
| import os | |||
| import h5py | |||
| import re | |||
| import datetime | |||
| from schainpy.model.data.jrodata import * | |||
| from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation | |||
George Yong
|
r1167 | # from .jroIO_base import * | |
|
Julio Valdez
|
r848 | from schainpy.model.io.jroIO_base import * | |
| import schainpy | |||
| class ParamReader(ProcessingUnit): | |||
| ''' | |||
| Reads HDF5 format files | |||
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | path | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | startDate | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | endDate | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | startTime | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | endTime | |
| ''' | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | ext = ".hdf5" | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | optchar = "D" | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | timezone = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | startTime = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | endTime = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | fileIndex = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | utcList = None #To select data in the utctime list | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | blockList = None #List to blocks to be read from the file | |
|
José Chávez
|
r897 | ||
| blocksPerFile = None #Number of blocks to be read | |||
|
Julio Valdez
|
r848 | blockIndex = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | path = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | #List of Files | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | filenameList = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | datetimeList = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | #Hdf5 File | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | listMetaname = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | listMeta = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | listDataname = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | listData = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | listShapes = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | fp = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | #dataOut reconstruction | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | dataOut = None | |
|
José Chávez
|
r897 | ||
| def __init__(self, **kwargs): | |||
| ProcessingUnit.__init__(self, **kwargs) | |||
|
Julio Valdez
|
r848 | self.dataOut = Parameters() | |
| return | |||
|
José Chávez
|
r897 | ||
| def setup(self, **kwargs): | |||
|
Julio Valdez
|
r848 | path = kwargs['path'] | |
| startDate = kwargs['startDate'] | |||
| endDate = kwargs['endDate'] | |||
| startTime = kwargs['startTime'] | |||
| endTime = kwargs['endTime'] | |||
| walk = kwargs['walk'] | |||
|
George Yong
|
r1167 | if 'ext' in kwargs: | |
|
Julio Valdez
|
r848 | ext = kwargs['ext'] | |
| else: | |||
| ext = '.hdf5' | |||
|
George Yong
|
r1167 | if 'timezone' in kwargs: | |
|
Julio Valdez
|
r848 | self.timezone = kwargs['timezone'] | |
| else: | |||
| self.timezone = 'lt' | |||
|
José Chávez
|
r897 | ||
|
George Yong
|
r1167 | print("[Reading] Searching files in offline mode ...") | |
Juan C. Espinoza
|
r1052 | pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate, | |
|
Julio Valdez
|
r848 | startTime=startTime, endTime=endTime, | |
|
José Chávez
|
r897 | ext=ext, walk=walk) | |
|
Julio Valdez
|
r848 | if not(filenameList): | |
|
George Yong
|
r1167 | print("There is no files into the folder: %s"%(path)) | |
|
Julio Valdez
|
r848 | sys.exit(-1) | |
| self.fileIndex = -1 | |||
| self.startTime = startTime | |||
| self.endTime = endTime | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | self.__readMetadata() | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | self.__setNextFileOffline() | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | return | |
|
José Chávez
|
r897 | ||
|
Juan C. Espinoza
|
r1052 | def searchFilesOffLine(self, | |
|
Julio Valdez
|
r848 | path, | |
| startDate=None, | |||
| endDate=None, | |||
| startTime=datetime.time(0,0,0), | |||
| endTime=datetime.time(23,59,59), | |||
| ext='.hdf5', | |||
| walk=True): | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | expLabel = '' | |
| self.filenameList = [] | |||
| self.datetimeList = [] | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | pathList = [] | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | JRODataObj = JRODataReader() | |
| dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if dateList == []: | |
|
George Yong
|
r1167 | print("[Reading] No *%s files in %s from %s to %s)"%(ext, path, | |
|
Julio Valdez
|
r848 | datetime.datetime.combine(startDate,startTime).ctime(), | |
|
George Yong
|
r1167 | datetime.datetime.combine(endDate,endTime).ctime())) | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | return None, None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if len(dateList) > 1: | |
|
George Yong
|
r1167 | print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)) | |
|
Julio Valdez
|
r848 | else: | |
|
George Yong
|
r1167 | print("[Reading] data was found for the date %s" %(dateList[0])) | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | filenameList = [] | |
| datetimeList = [] | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | #---------------------------------------------------------------------------------- | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | for thisPath in pathList: | |
| # thisPath = pathList[pathDict[file]] | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | fileList = glob.glob1(thisPath, "*%s" %ext) | |
| fileList.sort() | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | for file in fileList: | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | filename = os.path.join(thisPath,file) | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if not isFileInDateRange(filename, startDate, endDate): | |
| continue | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime) | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if not(thisDatetime): | |
| continue | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | filenameList.append(filename) | |
| datetimeList.append(thisDatetime) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if not(filenameList): | |
|
George Yong
|
r1167 | print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())) | |
|
Julio Valdez
|
r848 | return None, None | |
|
José Chávez
|
r897 | ||
|
George Yong
|
r1167 | print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)) | |
| print() | |||
|
José Chávez
|
r897 | ||
ebocanegra
|
r1001 | # for i in range(len(filenameList)): | |
| # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) | |||
|
Julio Valdez
|
r848 | ||
| self.filenameList = filenameList | |||
| self.datetimeList = datetimeList | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | return pathList, filenameList | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime): | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | """ | |
| Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | Inputs: | |
| filename : nombre completo del archivo de datos en formato Jicamarca (.r) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | startDate : fecha inicial del rango seleccionado en formato datetime.date | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | endDate : fecha final del rango seleccionado en formato datetime.date | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | startTime : tiempo inicial del rango seleccionado en formato datetime.time | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | endTime : tiempo final del rango seleccionado en formato datetime.time | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | Return: | |
| Boolean : Retorna True si el archivo de datos contiene datos en el rango de | |||
| fecha especificado, de lo contrario retorna False. | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | Excepciones: | |
| Si el archivo no existe o no puede ser abierto | |||
| Si la cabecera no puede ser leida. | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | """ | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | try: | |
| fp = h5py.File(filename,'r') | |||
| grp1 = fp['Data'] | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | except IOError: | |
| traceback.print_exc() | |||
|
George Yong
|
r1167 | raise IOError("The file %s can't be opened" %(filename)) | |
|
Julio Valdez
|
r848 | #chino rata | |
| #In case has utctime attribute | |||
| grp2 = grp1['utctime'] | |||
| # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time | |||
| thisUtcTime = grp2.value[0] | |||
| fp.close() | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if self.timezone == 'lt': | |
| thisUtcTime -= 5*3600 | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600) | |
| # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0]) | |||
| thisDate = thisDatetime.date() | |||
| thisTime = thisDatetime.time() | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds() | |
| endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds() | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | #General case | |
| # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o | |||
| #-----------o----------------------------o----------- | |||
| # startTime endTime | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if endTime >= startTime: | |
|
José Chávez
|
r897 | thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime) | |
|
Julio Valdez
|
r848 | if numpy.any(thisUtcLog): #If there is one block between the hours mentioned | |
| return thisDatetime | |||
| return None | |||
|
José Chávez
|
r897 | ||
| #If endTime < startTime then endTime belongs to the next day | |||
|
Julio Valdez
|
r848 | #<<<<<<<<<<<o o>>>>>>>>>>> | |
| #-----------o----------------------------o----------- | |||
| # endTime startTime | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime): | |
| return None | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime): | |
| return None | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime): | |
| return None | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | return thisDatetime | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | def __setNextFileOffline(self): | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | self.fileIndex += 1 | |
| idFile = self.fileIndex | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if not(idFile < len(self.filenameList)): | |
|
George Yong
|
r1167 | print("No more Files") | |
|
Julio Valdez
|
r848 | return 0 | |
| filename = self.filenameList[idFile] | |||
| filePointer = h5py.File(filename,'r') | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | self.filename = filename | |
| self.fp = filePointer | |||
|
George Yong
|
r1167 | print("Setting the file: %s"%self.filename) | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | # self.__readMetadata() | |
| self.__setBlockList() | |||
| self.__readData() | |||
| # self.nRecords = self.fp['Data'].attrs['blocksPerFile'] | |||
| # self.nRecords = self.fp['Data'].attrs['nRecords'] | |||
| self.blockIndex = 0 | |||
| return 1 | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | def __setBlockList(self): | |
| ''' | |||
| Selects the data within the times defined | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | self.fp | |
| self.startTime | |||
| self.endTime | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | self.blockList | |
| self.blocksPerFile | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | ''' | |
| fp = self.fp | |||
| startTime = self.startTime | |||
| endTime = self.endTime | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | grp = fp['Data'] | |
| thisUtcTime = grp['utctime'].value.astype(numpy.float)[0] | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | #ERROOOOR | |
| if self.timezone == 'lt': | |||
| thisUtcTime -= 5*3600 | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600) | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | thisDate = thisDatetime.date() | |
| thisTime = thisDatetime.time() | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds() | |
| endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds() | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0] | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | self.blockList = ind | |
| self.blocksPerFile = len(ind) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | return | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | def __readMetadata(self): | |
| ''' | |||
| Reads Metadata | |||
|
José Chávez
|
r897 | ||
| self.pathMeta | |||
|
Julio Valdez
|
r848 | self.listShapes | |
| self.listMetaname | |||
| self.listMeta | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | ''' | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | # grp = self.fp['Data'] | |
| # pathMeta = os.path.join(self.path, grp.attrs['metadata']) | |||
|
José Chávez
|
r897 | # | |
|
Julio Valdez
|
r848 | # if pathMeta == self.pathMeta: | |
| # return | |||
| # else: | |||
| # self.pathMeta = pathMeta | |||
|
José Chávez
|
r897 | # | |
|
Julio Valdez
|
r848 | # filePointer = h5py.File(self.pathMeta,'r') | |
| # groupPointer = filePointer['Metadata'] | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | filename = self.filenameList[0] | |
| fp = h5py.File(filename,'r') | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | gp = fp['Metadata'] | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | listMetaname = [] | |
| listMetadata = [] | |||
|
George Yong
|
r1167 | for item in list(gp.items()): | |
|
Julio Valdez
|
r848 | name = item[0] | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if name=='array dimensions': | |
| table = gp[name][:] | |||
| listShapes = {} | |||
| for shapes in table: | |||
| listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]]) | |||
| else: | |||
| data = gp[name].value | |||
| listMetaname.append(name) | |||
| listMetadata.append(data) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | # if name=='type': | |
| # self.__initDataOut(data) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | self.listShapes = listShapes | |
| self.listMetaname = listMetaname | |||
| self.listMeta = listMetadata | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | fp.close() | |
| return | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | def __readData(self): | |
| grp = self.fp['Data'] | |||
| listdataname = [] | |||
| listdata = [] | |||
|
José Chávez
|
r897 | ||
|
George Yong
|
r1167 | for item in list(grp.items()): | |
|
Julio Valdez
|
r848 | name = item[0] | |
| listdataname.append(name) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | array = self.__setDataArray(grp[name],self.listShapes[name]) | |
| listdata.append(array) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | self.listDataname = listdataname | |
| self.listData = listdata | |||
| return | |||
|
José Chávez
|
r897 | ||
|
Juan C. Valdez
|
r872 | def __setDataArray(self, dataset, shapes): | |
|
José Chávez
|
r897 | ||
| nDims = shapes[0] | |||
|
Juan C. Valdez
|
r872 | nDim2 = shapes[1] #Dimension 0 | |
|
José Chávez
|
r897 | ||
|
Juan C. Valdez
|
r872 | nDim1 = shapes[2] #Dimension 1, number of Points or Parameters | |
|
José Chávez
|
r897 | ||
|
Juan C. Valdez
|
r872 | nDim0 = shapes[3] #Dimension 2, number of samples or ranges | |
|
José Chávez
|
r897 | ||
|
Juan C. Valdez
|
r872 | mode = shapes[4] #Mode of storing | |
|
José Chávez
|
r897 | ||
|
Juan C. Valdez
|
r872 | blockList = self.blockList | |
|
José Chávez
|
r897 | ||
|
Juan C. Valdez
|
r872 | blocksPerFile = self.blocksPerFile | |
|
José Chávez
|
r897 | ||
|
Juan C. Valdez
|
r872 | #Depending on what mode the data was stored | |
| if mode == 0: #Divided in channels | |||
| arrayData = dataset.value.astype(numpy.float)[0][blockList] | |||
| if mode == 1: #Divided in parameter | |||
| strds = 'table' | |||
| nDatas = nDim1 | |||
| newShapes = (blocksPerFile,nDim2,nDim0) | |||
| elif mode==2: #Concatenated in a table | |||
| strds = 'table0' | |||
| arrayData = dataset[strds].value | |||
| #Selecting part of the dataset | |||
| utctime = arrayData[:,0] | |||
| u, indices = numpy.unique(utctime, return_index=True) | |||
|
José Chávez
|
r897 | ||
|
Juan C. Valdez
|
r872 | if blockList.size != indices.size: | |
| indMin = indices[blockList[0]] | |||
| if blockList[1] + 1 >= indices.size: | |||
| arrayData = arrayData[indMin:,:] | |||
| else: | |||
| indMax = indices[blockList[1] + 1] | |||
| arrayData = arrayData[indMin:indMax,:] | |||
| return arrayData | |||
|
José Chávez
|
r897 | ||
| # One dimension | |||
|
Juan C. Valdez
|
r872 | if nDims == 0: | |
| arrayData = dataset.value.astype(numpy.float)[0][blockList] | |||
|
José Chávez
|
r897 | ||
| # Two dimensions | |||
|
Juan C. Valdez
|
r872 | elif nDims == 2: | |
| arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0)) | |||
| newShapes = (blocksPerFile,nDim0) | |||
|
José Chávez
|
r897 | nDatas = nDim1 | |
| for i in range(nDatas): | |||
|
Juan C. Valdez
|
r872 | data = dataset[strds + str(i)].value | |
|
José Chávez
|
r897 | arrayData[:,i,:] = data[blockList,:] | |
| # Three dimensions | |||
|
Juan C. Valdez
|
r872 | else: | |
|
José Chávez
|
r897 | arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0)) | |
|
Juan C. Valdez
|
r872 | for i in range(nDatas): | |
|
José Chávez
|
r897 | ||
|
Juan C. Valdez
|
r872 | data = dataset[strds + str(i)].value | |
|
José Chávez
|
r897 | ||
|
Juan C. Valdez
|
r872 | for b in range(blockList.size): | |
| arrayData[b,:,i,:] = data[:,:,blockList[b]] | |||
|
José Chávez
|
r897 | ||
| return arrayData | |||
|
Julio Valdez
|
r848 | def __setDataOut(self): | |
| listMeta = self.listMeta | |||
| listMetaname = self.listMetaname | |||
| listDataname = self.listDataname | |||
| listData = self.listData | |||
| listShapes = self.listShapes | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | blockIndex = self.blockIndex | |
| # blockList = self.blockList | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | for i in range(len(listMeta)): | |
| setattr(self.dataOut,listMetaname[i],listMeta[i]) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | for j in range(len(listData)): | |
| nShapes = listShapes[listDataname[j]][0] | |||
| mode = listShapes[listDataname[j]][4] | |||
| if nShapes == 1: | |||
| setattr(self.dataOut,listDataname[j],listData[j][blockIndex]) | |||
| elif nShapes > 1: | |||
| setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:]) | |||
| elif mode==0: | |||
| setattr(self.dataOut,listDataname[j],listData[j][blockIndex]) | |||
| #Mode Meteors | |||
| elif mode ==2: | |||
| selectedData = self.__selectDataMode2(listData[j], blockIndex) | |||
| setattr(self.dataOut, listDataname[j], selectedData) | |||
| return | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | def __selectDataMode2(self, data, blockIndex): | |
| utctime = data[:,0] | |||
| aux, indices = numpy.unique(utctime, return_inverse=True) | |||
| selInd = numpy.where(indices == blockIndex)[0] | |||
| selData = data[selInd,:] | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | return selData | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | def getData(self): | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | # if self.flagNoMoreFiles: | |
| # self.dataOut.flagNoData = True | |||
| # print 'Process finished' | |||
| # return 0 | |||
|
José Chávez
|
r897 | # | |
|
Julio Valdez
|
r848 | if self.blockIndex==self.blocksPerFile: | |
| if not( self.__setNextFileOffline() ): | |||
| self.dataOut.flagNoData = True | |||
| return 0 | |||
| # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers | |||
|
José Chávez
|
r897 | # self.dataOut.flagNoData = True | |
|
Julio Valdez
|
r848 | # return 0 | |
| # self.__readData() | |||
| self.__setDataOut() | |||
| self.dataOut.flagNoData = False | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | self.blockIndex += 1 | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | return | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | def run(self, **kwargs): | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if not(self.isConfig): | |
| self.setup(**kwargs) | |||
| # self.setObjProperties() | |||
| self.isConfig = True | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | self.getData() | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | return | |
| class ParamWriter(Operation): | |||
| ''' | |||
|
José Chávez
|
r897 | HDF5 Writer, stores parameters data in HDF5 format files | |
|
Julio Valdez
|
r848 | path: path where the files will be stored | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | blocksPerFile: number of blocks that will be saved in per HDF5 format file | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors) | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | metadataList: list of attributes that will be stored as metadata | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | dataList: list of attributes that will be stores as data | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | ''' | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | ext = ".hdf5" | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | optchar = "D" | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | metaoptchar = "M" | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | metaFile = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | filename = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | path = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | setFile = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | fp = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | grp = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | ds = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | firsttime = True | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | #Configurations | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | blocksPerFile = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | blockIndex = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | dataOut = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | #Data Arrays | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | dataList = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | metadataList = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | # arrayDim = None | |
| dsList = None #List of dictionaries with dataset properties | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | tableDim = None | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')] | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')] | |
| currentDay = None | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r853 | lastTime = None | |
|
José Chávez
|
r897 | ||
| def __init__(self, **kwargs): | |||
| Operation.__init__(self, **kwargs) | |||
|
Julio Valdez
|
r848 | self.isConfig = False | |
| return | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r955 | def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs): | |
| self.path = path | |||
| self.blocksPerFile = blocksPerFile | |||
| self.metadataList = metadataList | |||
| self.dataList = dataList | |||
|
Julio Valdez
|
r848 | self.dataOut = dataOut | |
| self.mode = mode | |||
|
José Chávez
|
r955 | ||
| if self.mode is not None: | |||
| self.mode = numpy.zeros(len(self.dataList)) + mode | |||
| else: | |||
| self.mode = numpy.ones(len(self.dataList)) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | arrayDim = numpy.zeros((len(self.dataList),5)) | |
|
José Chávez
|
r897 | ||
| #Table dimensions | |||
|
Julio Valdez
|
r848 | dtype0 = self.dtype | |
| tableList = [] | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | #Dictionary and list of tables | |
| dsList = [] | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | for i in range(len(self.dataList)): | |
| dsDict = {} | |||
| dataAux = getattr(self.dataOut, self.dataList[i]) | |||
| dsDict['variable'] = self.dataList[i] | |||
| #--------------------- Conditionals ------------------------ | |||
| #There is no data | |||
|
José Chávez
|
r897 | if dataAux is None: | |
|
Julio Valdez
|
r848 | return 0 | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | #Not array, just a number | |
| #Mode 0 | |||
| if type(dataAux)==float or type(dataAux)==int: | |||
| dsDict['mode'] = 0 | |||
| dsDict['nDim'] = 0 | |||
| arrayDim[i,0] = 0 | |||
| dsList.append(dsDict) | |||
| #Mode 2: meteors | |||
| elif mode[i] == 2: | |||
|
José Chávez
|
r897 | # dsDict['nDim'] = 0 | |
| dsDict['dsName'] = 'table0' | |||
|
Julio Valdez
|
r848 | dsDict['mode'] = 2 # Mode meteors | |
| dsDict['shape'] = dataAux.shape[-1] | |||
| dsDict['nDim'] = 0 | |||
| dsDict['dsNumber'] = 1 | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | arrayDim[i,3] = dataAux.shape[-1] | |
| arrayDim[i,4] = mode[i] #Mode the data was stored | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | dsList.append(dsDict) | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | #Mode 1 | |
| else: | |||
| arrayDim0 = dataAux.shape #Data dimensions | |||
| arrayDim[i,0] = len(arrayDim0) #Number of array dimensions | |||
| arrayDim[i,4] = mode[i] #Mode the data was stored | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | strtable = 'table' | |
| dsDict['mode'] = 1 # Mode parameters | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | # Three-dimension arrays | |
| if len(arrayDim0) == 3: | |||
| arrayDim[i,1:-1] = numpy.array(arrayDim0) | |||
| nTables = int(arrayDim[i,2]) | |||
| dsDict['dsNumber'] = nTables | |||
| dsDict['shape'] = arrayDim[i,2:4] | |||
| dsDict['nDim'] = 3 | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | for j in range(nTables): | |
| dsDict = dsDict.copy() | |||
| dsDict['dsName'] = strtable + str(j) | |||
| dsList.append(dsDict) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | # Two-dimension arrays | |
| elif len(arrayDim0) == 2: | |||
| arrayDim[i,2:-1] = numpy.array(arrayDim0) | |||
| nTables = int(arrayDim[i,2]) | |||
| dsDict['dsNumber'] = nTables | |||
| dsDict['shape'] = arrayDim[i,3] | |||
| dsDict['nDim'] = 2 | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | for j in range(nTables): | |
| dsDict = dsDict.copy() | |||
| dsDict['dsName'] = strtable + str(j) | |||
| dsList.append(dsDict) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | # One-dimension arrays | |
| elif len(arrayDim0) == 1: | |||
| arrayDim[i,3] = arrayDim0[0] | |||
| dsDict['shape'] = arrayDim0[0] | |||
| dsDict['dsNumber'] = 1 | |||
| dsDict['dsName'] = strtable + str(0) | |||
| dsDict['nDim'] = 1 | |||
| dsList.append(dsDict) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0) | |
| tableList.append(table) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | # self.arrayDim = arrayDim | |
|
José Chávez
|
r897 | self.dsList = dsList | |
| self.tableDim = numpy.array(tableList, dtype = dtype0) | |||
|
Julio Valdez
|
r848 | self.blockIndex = 0 | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | timeTuple = time.localtime(dataOut.utctime) | |
| self.currentDay = timeTuple.tm_yday | |||
| return 1 | |||
| def putMetadata(self): | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | fp = self.createMetadataFile() | |
|
José Chávez
|
r897 | self.writeMetadata(fp) | |
|
Julio Valdez
|
r848 | fp.close() | |
| return | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | def createMetadataFile(self): | |
| ext = self.ext | |||
| path = self.path | |||
| setFile = self.setFile | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | timeTuple = time.localtime(self.dataOut.utctime) | |
|
José Chávez
|
r897 | ||
| subfolder = '' | |||
|
Julio Valdez
|
r848 | fullpath = os.path.join( path, subfolder ) | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if not( os.path.exists(fullpath) ): | |
| os.mkdir(fullpath) | |||
| setFile = -1 #inicializo mi contador de seteo | |||
|
José Chávez
|
r897 | ||
| subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) | |||
|
Julio Valdez
|
r848 | fullpath = os.path.join( path, subfolder ) | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if not( os.path.exists(fullpath) ): | |
| os.mkdir(fullpath) | |||
|
José Chávez
|
r897 | setFile = -1 #inicializo mi contador de seteo | |
|
Julio Valdez
|
r848 | ||
| else: | |||
| filesList = os.listdir( fullpath ) | |||
| filesList = sorted( filesList, key=str.lower ) | |||
| if len( filesList ) > 0: | |||
| filesList = [k for k in filesList if 'M' in k] | |||
| filen = filesList[-1] | |||
| # el filename debera tener el siguiente formato | |||
| # 0 1234 567 89A BCDE (hex) | |||
| # x YYYY DDD SSS .ext | |||
| if isNumber( filen[8:11] ): | |||
| setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file | |||
|
José Chávez
|
r897 | else: | |
|
Julio Valdez
|
r848 | setFile = -1 | |
| else: | |||
| setFile = -1 #inicializo mi contador de seteo | |||
|
Juan C. Espinoza
|
r986 | if self.setType is None: | |
| setFile += 1 | |||
| file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar, | |||
| timeTuple.tm_year, | |||
| timeTuple.tm_yday, | |||
| setFile, | |||
| ext ) | |||
| else: | |||
| setFile = timeTuple.tm_hour*60+timeTuple.tm_min | |||
| file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar, | |||
| timeTuple.tm_year, | |||
| timeTuple.tm_yday, | |||
| setFile, | |||
| ext ) | |||
|
Julio Valdez
|
r848 | ||
| filename = os.path.join( path, subfolder, file ) | |||
| self.metaFile = file | |||
| #Setting HDF5 File | |||
| fp = h5py.File(filename,'w') | |||
| return fp | |||
|
José Chávez
|
r897 | ||
| def writeMetadata(self, fp): | |||
|
Julio Valdez
|
r848 | grp = fp.create_group("Metadata") | |
| grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | for i in range(len(self.metadataList)): | |
| grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i])) | |||
| return | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r853 | def timeFlag(self): | |
| currentTime = self.dataOut.utctime | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r853 | if self.lastTime is None: | |
| self.lastTime = currentTime | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r853 | #Day | |
| timeTuple = time.localtime(currentTime) | |||
|
Julio Valdez
|
r848 | dataDay = timeTuple.tm_yday | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r853 | #Time | |
| timeDiff = currentTime - self.lastTime | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r853 | #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora | |
| if dataDay != self.currentDay: | |||
| self.currentDay = dataDay | |||
| return True | |||
| elif timeDiff > 3*60*60: | |||
| self.lastTime = currentTime | |||
| return True | |||
| else: | |||
| self.lastTime = currentTime | |||
| return False | |||
|
Julio Valdez
|
r848 | ||
| def setNextFile(self): | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | ext = self.ext | |
| path = self.path | |||
| setFile = self.setFile | |||
| mode = self.mode | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | timeTuple = time.localtime(self.dataOut.utctime) | |
| subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) | |||
| fullpath = os.path.join( path, subfolder ) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if os.path.exists(fullpath): | |
| filesList = os.listdir( fullpath ) | |||
| filesList = [k for k in filesList if 'D' in k] | |||
| if len( filesList ) > 0: | |||
| filesList = sorted( filesList, key=str.lower ) | |||
| filen = filesList[-1] | |||
| # el filename debera tener el siguiente formato | |||
| # 0 1234 567 89A BCDE (hex) | |||
| # x YYYY DDD SSS .ext | |||
| if isNumber( filen[8:11] ): | |||
| setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file | |||
|
José Chávez
|
r897 | else: | |
|
Julio Valdez
|
r848 | setFile = -1 | |
| else: | |||
| setFile = -1 #inicializo mi contador de seteo | |||
| else: | |||
|
Juan C. Espinoza
|
r928 | os.makedirs(fullpath) | |
|
José Chávez
|
r897 | setFile = -1 #inicializo mi contador de seteo | |
|
Juan C. Espinoza
|
r986 | if self.setType is None: | |
| setFile += 1 | |||
| file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar, | |||
| timeTuple.tm_year, | |||
| timeTuple.tm_yday, | |||
| setFile, | |||
| ext ) | |||
| else: | |||
| setFile = timeTuple.tm_hour*60+timeTuple.tm_min | |||
| file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar, | |||
| timeTuple.tm_year, | |||
| timeTuple.tm_yday, | |||
| setFile, | |||
| ext ) | |||
|
Julio Valdez
|
r848 | ||
| filename = os.path.join( path, subfolder, file ) | |||
| #Setting HDF5 File | |||
| fp = h5py.File(filename,'w') | |||
| #write metadata | |||
|
José Chávez
|
r897 | self.writeMetadata(fp) | |
|
Julio Valdez
|
r848 | #Write data | |
| grp = fp.create_group("Data") | |||
| # grp.attrs['metadata'] = self.metaFile | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | # grp.attrs['blocksPerFile'] = 0 | |
| ds = [] | |||
| data = [] | |||
| dsList = self.dsList | |||
| i = 0 | |||
|
José Chávez
|
r897 | while i < len(dsList): | |
|
Julio Valdez
|
r848 | dsInfo = dsList[i] | |
| #One-dimension data | |||
| if dsInfo['mode'] == 0: | |||
| # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20') | |||
| ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64) | |||
| ds.append(ds0) | |||
| data.append([]) | |||
| i += 1 | |||
| continue | |||
| # nDimsForDs.append(nDims[i]) | |||
| elif dsInfo['mode'] == 2: | |||
| grp0 = grp.create_group(dsInfo['variable']) | |||
| ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True) | |||
| ds.append(ds0) | |||
| data.append([]) | |||
| i += 1 | |||
| continue | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | elif dsInfo['mode'] == 1: | |
| grp0 = grp.create_group(dsInfo['variable']) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | for j in range(dsInfo['dsNumber']): | |
| dsInfo = dsList[i] | |||
| tableName = dsInfo['dsName'] | |||
|
Juan C. Valdez
|
r881 | shape = int(dsInfo['shape']) | |
|
José Chávez
|
r897 | ||
| if dsInfo['nDim'] == 3: | |||
|
Julio Valdez
|
r848 | ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True) | |
| else: | |||
| ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | ds.append(ds0) | |
| data.append([]) | |||
| i += 1 | |||
| # nDimsForDs.append(nDims[i]) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | fp.flush() | |
| fp.close() | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | # self.nDatas = nDatas | |
| # self.nDims = nDims | |||
| # self.nDimsForDs = nDimsForDs | |||
|
José Chávez
|
r897 | #Saving variables | |
|
George Yong
|
r1167 | print('Writing the file: %s'%filename) | |
|
Julio Valdez
|
r848 | self.filename = filename | |
| # self.fp = fp | |||
| # self.grp = grp | |||
| # self.grp.attrs.modify('nRecords', 1) | |||
| self.ds = ds | |||
| self.data = data | |||
| # self.setFile = setFile | |||
| self.firsttime = True | |||
| self.blockIndex = 0 | |||
| return | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | def putData(self): | |
|
Julio Valdez
|
r853 | if self.blockIndex == self.blocksPerFile or self.timeFlag(): | |
|
José Chávez
|
r897 | self.setNextFile() | |
|
Julio Valdez
|
r848 | # if not self.firsttime: | |
| self.readBlock() | |||
| self.setBlock() #Prepare data to be written | |||
| self.writeBlock() #Write data | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | return | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | def readBlock(self): | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | ''' | |
| data Array configured | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | self.data | |
| ''' | |||
| dsList = self.dsList | |||
| ds = self.ds | |||
| #Setting HDF5 File | |||
| fp = h5py.File(self.filename,'r+') | |||
| grp = fp["Data"] | |||
| ind = 0 | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | # grp.attrs['blocksPerFile'] = 0 | |
|
José Chávez
|
r897 | while ind < len(dsList): | |
|
Julio Valdez
|
r848 | dsInfo = dsList[ind] | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if dsInfo['mode'] == 0: | |
| ds0 = grp[dsInfo['variable']] | |||
| ds[ind] = ds0 | |||
| ind += 1 | |||
| else: | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | grp0 = grp[dsInfo['variable']] | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | for j in range(dsInfo['dsNumber']): | |
| dsInfo = dsList[ind] | |||
| ds0 = grp0[dsInfo['dsName']] | |||
| ds[ind] = ds0 | |||
| ind += 1 | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | self.fp = fp | |
| self.grp = grp | |||
| self.ds = ds | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | return | |
| def setBlock(self): | |||
| ''' | |||
| data Array configured | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | self.data | |
| ''' | |||
| #Creating Arrays | |||
| dsList = self.dsList | |||
| data = self.data | |||
| ind = 0 | |||
|
José Chávez
|
r897 | ||
| while ind < len(dsList): | |||
|
Julio Valdez
|
r848 | dsInfo = dsList[ind] | |
| dataAux = getattr(self.dataOut, dsInfo['variable']) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | mode = dsInfo['mode'] | |
| nDim = dsInfo['nDim'] | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if mode == 0 or mode == 2 or nDim == 1: | |
| data[ind] = dataAux | |||
|
José Chávez
|
r897 | ind += 1 | |
|
Julio Valdez
|
r848 | # elif nDim == 1: | |
| # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1)) | |||
| # ind += 1 | |||
| elif nDim == 2: | |||
| for j in range(dsInfo['dsNumber']): | |||
| data[ind] = dataAux[j,:] | |||
| ind += 1 | |||
| elif nDim == 3: | |||
| for j in range(dsInfo['dsNumber']): | |||
| data[ind] = dataAux[:,j,:] | |||
| ind += 1 | |||
| self.data = data | |||
| return | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | def writeBlock(self): | |
| ''' | |||
| Saves the block in the HDF5 file | |||
| ''' | |||
|
José Chávez
|
r897 | dsList = self.dsList | |
|
Julio Valdez
|
r848 | for i in range(len(self.ds)): | |
|
José Chávez
|
r897 | dsInfo = dsList[i] | |
|
Julio Valdez
|
r848 | nDim = dsInfo['nDim'] | |
| mode = dsInfo['mode'] | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | # First time | |
| if self.firsttime: | |||
| # self.ds[i].resize(self.data[i].shape) | |||
| # self.ds[i][self.blockIndex,:] = self.data[i] | |||
| if type(self.data[i]) == numpy.ndarray: | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if nDim == 3: | |
| self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1)) | |||
| self.ds[i].resize(self.data[i].shape) | |||
| if mode == 2: | |||
| self.ds[i].resize(self.data[i].shape) | |||
|
José Chávez
|
r897 | self.ds[i][:] = self.data[i] | |
| else: | |||
|
Julio Valdez
|
r848 | # From second time | |
| # Meteors! | |||
| if mode == 2: | |||
| dataShape = self.data[i].shape | |||
| dsShape = self.ds[i].shape | |||
|
José Chávez
|
r897 | self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1])) | |
|
Julio Valdez
|
r848 | self.ds[i][dsShape[0]:,:] = self.data[i] | |
| # No dimension | |||
| elif mode == 0: | |||
| self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1)) | |||
| self.ds[i][0,-1] = self.data[i] | |||
| # One dimension | |||
| elif nDim == 1: | |||
| self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1])) | |||
|
José Chávez
|
r897 | self.ds[i][-1,:] = self.data[i] | |
|
Julio Valdez
|
r848 | # Two dimension | |
| elif nDim == 2: | |||
|
José Chávez
|
r897 | self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1])) | |
|
Julio Valdez
|
r848 | self.ds[i][self.blockIndex,:] = self.data[i] | |
| # Three dimensions | |||
| elif nDim == 3: | |||
|
José Chávez
|
r897 | self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1)) | |
|
Julio Valdez
|
r848 | self.ds[i][:,:,-1] = self.data[i] | |
|
José Chávez
|
r897 | ||
| self.firsttime = False | |||
|
Julio Valdez
|
r848 | self.blockIndex += 1 | |
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | #Close to save changes | |
| self.fp.flush() | |||
| self.fp.close() | |||
| return | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r955 | def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs): | |
|
Julio Valdez
|
r848 | ||
| if not(self.isConfig): | |||
|
José Chávez
|
r955 | flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile, | |
| metadataList=metadataList, dataList=dataList, mode=mode, **kwargs) | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | if not(flagdata): | |
| return | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | self.isConfig = True | |
| # self.putMetadata() | |||
| self.setNextFile() | |||
|
José Chávez
|
r897 | ||
|
Julio Valdez
|
r848 | self.putData() | |
|
George Yong
|
r1167 | return |