jroIO_param.py
748 lines
| 23.7 KiB
| text/x-python
|
PythonLexer
Juan C. Espinoza
|
r1536 | from email.utils import localtime | ||
Julio Valdez
|
r848 | import os | ||
|
Juan C. Espinoza
|
r1326 | import time | ||
|
Julio Valdez
|
r848 | import datetime | ||
|
Juan C. Espinoza
|
r1326 | import numpy | ||
| import h5py | ||||
| r1241 | import schainpy.admin | |||
|
Julio Valdez
|
r848 | from schainpy.model.data.jrodata import * | ||
George Yong
|
r1179 | from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator | ||
|
Julio Valdez
|
r848 | from schainpy.model.io.jroIO_base import * | ||
|
George Yong
|
r1179 | from schainpy.utils import log | ||
|
Julio Valdez
|
r848 | |||
|
Juan C. Espinoza
|
r1287 | |||
|
Juan C. Espinoza
|
r1326 | class HDFReader(Reader, ProcessingUnit): | ||
| """Processing unit to read HDF5 format files | ||||
| This unit reads HDF5 files created with `HDFWriter` operation contains | ||||
| by default two groups Data and Metadata all variables would be saved as `dataOut` | ||||
avaldezp
|
r1405 | attributes. | ||
|
Juan C. Espinoza
|
r1326 | It is possible to read any HDF5 file by given the structure in the `description` | ||
| parameter, also you can add extra values to metadata with the parameter `extras`. | ||||
| Parameters: | ||||
| ----------- | ||||
| path : str | ||||
| Path where files are located. | ||||
| startDate : date | ||||
| Start date of the files | ||||
| endDate : list | ||||
| End date of the files | ||||
| startTime : time | ||||
| Start time of the files | ||||
| endTime : time | ||||
| End time of the files | ||||
| description : dict, optional | ||||
| Dictionary with the description of the HDF5 file | ||||
| extras : dict, optional | ||||
| Dictionary with extra metadata to be be added to `dataOut` | ||||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1326 | Examples | ||
| -------- | ||||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1326 | desc = { | ||
| 'Data': { | ||||
| 'data_output': ['u', 'v', 'w'], | ||||
| 'utctime': 'timestamps', | ||||
| } , | ||||
| 'Metadata': { | ||||
| 'heightList': 'heights' | ||||
| } | ||||
| } | ||||
| desc = { | ||||
| 'Data': { | ||||
| 'data_output': 'winds', | ||||
| 'utctime': 'timestamps' | ||||
| }, | ||||
| 'Metadata': { | ||||
| 'heightList': 'heights' | ||||
| } | ||||
| } | ||||
| extras = { | ||||
| 'timeZone': 300 | ||||
| } | ||||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1326 | reader = project.addReadUnit( | ||
| name='HDFReader', | ||||
| path='/path/to/files', | ||||
| startDate='2019/01/01', | ||||
| endDate='2019/01/31', | ||||
| startTime='00:00:00', | ||||
| endTime='23:59:59', | ||||
| # description=json.dumps(desc), | ||||
| # extras=json.dumps(extras), | ||||
| ) | ||||
|
Juan C. Espinoza
|
r1232 | |||
|
Juan C. Espinoza
|
r1326 | """ | ||
|
Juan C. Espinoza
|
r1287 | |||
|
Juan C. Espinoza
|
r1326 | __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras'] | ||
|
Juan C. Espinoza
|
r1232 | |||
| def __init__(self): | ||||
| ProcessingUnit.__init__(self) | ||||
| self.dataOut = Parameters() | ||||
| r1254 | self.ext = ".hdf5" | |||
| self.optchar = "D" | ||||
|
Juan C. Espinoza
|
r1326 | self.meta = {} | ||
| self.data = {} | ||||
| r1254 | self.open_file = h5py.File | |||
| self.open_mode = 'r' | ||||
|
Juan C. Espinoza
|
r1326 | self.description = {} | ||
| self.extras = {} | ||||
| r1254 | self.filefmt = "*%Y%j***" | |||
| self.folderfmt = "*%Y%j" | ||||
| r1363 | self.utcoffset = 0 | |||
|
avaldezp
|
r1534 | self.filter = None | ||
| self.dparam = None | ||||
|
Juan C. Espinoza
|
r1232 | |||
| def setup(self, **kwargs): | ||||
| r1254 | self.set_kwargs(**kwargs) | |||
| if not self.ext.startswith('.'): | ||||
|
avaldezp
|
r1405 | self.ext = '.{}'.format(self.ext) | ||
|
Juan C. Espinoza
|
r1232 | |||
| r1254 | if self.online: | |||
| log.log("Searching files in online mode...", self.name) | ||||
|
Juan C. Espinoza
|
r1232 | |||
| r1254 | for nTries in range(self.nTries): | |||
| fullpath = self.searchFilesOnLine(self.path, self.startDate, | ||||
|
avaldezp
|
r1405 | self.endDate, self.expLabel, self.ext, self.walk, | ||
|
avaldezp
|
r1534 | self.filefmt, self.folderfmt,self.filter) | ||
| r1254 | try: | |||
| fullpath = next(fullpath) | ||||
| except: | ||||
| fullpath = None | ||||
|
avaldezp
|
r1405 | |||
| r1254 | if fullpath: | |||
| break | ||||
| log.warning( | ||||
| 'Waiting {} sec for a valid file in {}: try {} ...'.format( | ||||
|
avaldezp
|
r1405 | self.delay, self.path, nTries + 1), | ||
| r1254 | self.name) | |||
| time.sleep(self.delay) | ||||
| if not(fullpath): | ||||
| raise schainpy.admin.SchainError( | ||||
|
avaldezp
|
r1405 | 'There isn\'t any valid file in {}'.format(self.path)) | ||
| r1254 | ||||
| pathname, filename = os.path.split(fullpath) | ||||
| self.year = int(filename[1:5]) | ||||
| self.doy = int(filename[5:8]) | ||||
|
avaldezp
|
r1405 | self.set = int(filename[8:11]) - 1 | ||
|
Juan C. Espinoza
|
r1232 | else: | ||
| r1254 | log.log("Searching files in {}".format(self.path), self.name) | |||
|
avaldezp
|
r1405 | self.filenameList = self.searchFilesOffLine(self.path, self.startDate, | ||
|
avaldezp
|
r1534 | self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt,self.filter) | ||
|
avaldezp
|
r1405 | |||
| r1254 | self.setNextFile() | |||
|
Juan C. Espinoza
|
r1232 | |||
| r1254 | return | |||
|
Juan C. Espinoza
|
r1232 | |||
| r1254 | def readFirstHeader(self): | |||
| '''Read metadata and data''' | ||||
|
Juan C. Espinoza
|
r1232 | |||
|
avaldezp
|
r1405 | self.__readMetadata() | ||
|
Juan C. Espinoza
|
r1232 | self.__readData() | ||
| r1254 | self.__setBlockList() | |||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1326 | if 'type' in self.meta: | ||
| self.dataOut = eval(self.meta['type'])() | ||||
|
avaldezp
|
r1405 | |||
|
avaldezp
|
r1534 | if self.dparam: | ||
| setattr(self.dataOut, "dparam", 1) | ||||
|
Juan C. Espinoza
|
r1326 | for attr in self.meta: | ||
| setattr(self.dataOut, attr, self.meta[attr]) | ||||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1232 | self.blockIndex = 0 | ||
|
avaldezp
|
r1405 | |||
| r1254 | return | |||
|
Juan C. Espinoza
|
r1232 | |||
| def __setBlockList(self): | ||||
| ''' | ||||
| Selects the data within the times defined | ||||
| self.fp | ||||
| self.startTime | ||||
| self.endTime | ||||
| self.blockList | ||||
| self.blocksPerFile | ||||
| ''' | ||||
| r1254 | ||||
|
Juan C. Espinoza
|
r1232 | startTime = self.startTime | ||
| endTime = self.endTime | ||||
| r1363 | thisUtcTime = self.data['utctime'] + self.utcoffset | |||
|
avaldezp
|
r1534 | try: | ||
| self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1]) | ||||
| except: | ||||
| self.interval = 0 | ||||
|
Juan C. Espinoza
|
r1326 | thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0]) | ||
|
Juan C. Espinoza
|
r1232 | |||
| thisDate = thisDatetime.date() | ||||
| thisTime = thisDatetime.time() | ||||
|
Juan C. Espinoza
|
r1326 | startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds() | ||
| endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds() | ||||
|
Juan C. Espinoza
|
r1232 | |||
| ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0] | ||||
| self.blockList = ind | ||||
| self.blocksPerFile = len(ind) | ||||
| return | ||||
| def __readMetadata(self): | ||||
| ''' | ||||
| Reads Metadata | ||||
| ''' | ||||
|
Juan C. Espinoza
|
r1326 | meta = {} | ||
| if self.description: | ||||
| for key, value in self.description['Metadata'].items(): | ||||
| r1360 | meta[key] = self.fp[value][()] | |||
| r1254 | else: | |||
|
Juan C. Espinoza
|
r1326 | grp = self.fp['Metadata'] | ||
| for name in grp: | ||||
| r1360 | meta[name] = grp[name][()] | |||
|
Juan C. Espinoza
|
r1232 | |||
|
Juan C. Espinoza
|
r1326 | if self.extras: | ||
| for key, value in self.extras.items(): | ||||
| meta[key] = value | ||||
| self.meta = meta | ||||
|
Juan C. Espinoza
|
r1232 | |||
| return | ||||
| def __readData(self): | ||||
|
Juan C. Espinoza
|
r1326 | data = {} | ||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1326 | if self.description: | ||
| for key, value in self.description['Data'].items(): | ||||
| if isinstance(value, str): | ||||
| if isinstance(self.fp[value], h5py.Dataset): | ||||
| r1360 | data[key] = self.fp[value][()] | |||
|
Juan C. Espinoza
|
r1326 | elif isinstance(self.fp[value], h5py.Group): | ||
| array = [] | ||||
| for ch in self.fp[value]: | ||||
| r1360 | array.append(self.fp[value][ch][()]) | |||
|
Juan C. Espinoza
|
r1326 | data[key] = numpy.array(array) | ||
| elif isinstance(value, list): | ||||
| array = [] | ||||
| for ch in value: | ||||
| r1360 | array.append(self.fp[ch][()]) | |||
|
Juan C. Espinoza
|
r1326 | data[key] = numpy.array(array) | ||
| else: | ||||
| r1254 | grp = self.fp['Data'] | |||
|
Juan C. Espinoza
|
r1326 | for name in grp: | ||
| if isinstance(grp[name], h5py.Dataset): | ||||
| r1360 | array = grp[name][()] | |||
|
Juan C. Espinoza
|
r1326 | elif isinstance(grp[name], h5py.Group): | ||
| r1254 | array = [] | |||
|
Juan C. Espinoza
|
r1326 | for ch in grp[name]: | ||
| r1360 | array.append(grp[name][ch][()]) | |||
| r1254 | array = numpy.array(array) | |||
| else: | ||||
|
Juan C. Espinoza
|
r1326 | log.warning('Unknown type: {}'.format(name)) | ||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1326 | if name in self.description: | ||
| key = self.description[name] | ||||
| else: | ||||
| key = name | ||||
| data[key] = array | ||||
|
Juan C. Espinoza
|
r1232 | |||
|
Juan C. Espinoza
|
r1326 | self.data = data | ||
|
Juan C. Espinoza
|
r1232 | return | ||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1232 | def getData(self): | ||
|
Juan C. Espinoza
|
r1326 | for attr in self.data: | ||
| if self.data[attr].ndim == 1: | ||||
| setattr(self.dataOut, attr, self.data[attr][self.blockIndex]) | ||||
|
Juan C. Espinoza
|
r1232 | else: | ||
|
avaldezp
|
r1534 | if self.dparam: | ||
| setattr(self.dataOut, attr, self.data[attr]) | ||||
| else: | ||||
| setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex]) | ||||
|
Juan C. Espinoza
|
r1232 | |||
| self.dataOut.flagNoData = False | ||||
| self.blockIndex += 1 | ||||
|
Juan C. Espinoza
|
r1326 | log.log("Block No. {}/{} -> {}".format( | ||
| self.blockIndex, | ||||
| self.blocksPerFile, | ||||
| self.dataOut.datatime.ctime()), self.name) | ||||
|
Juan C. Espinoza
|
r1232 | return | ||
| def run(self, **kwargs): | ||||
| if not(self.isConfig): | ||||
| self.setup(**kwargs) | ||||
| self.isConfig = True | ||||
| if self.blockIndex == self.blocksPerFile: | ||||
| r1254 | self.setNextFile() | |||
|
Juan C. Espinoza
|
r1232 | |||
| self.getData() | ||||
| return | ||||
| @MPDecorator | ||||
|
Juan C. Espinoza
|
r1326 | class HDFWriter(Operation): | ||
| """Operation to write HDF5 files. | ||||
| The HDF5 file contains by default two groups Data and Metadata where | ||||
| you can save any `dataOut` attribute specified by `dataList` and `metadataList` | ||||
| parameters, data attributes are normaly time dependent where the metadata | ||||
|
avaldezp
|
r1405 | are not. | ||
| It is possible to customize the structure of the HDF5 file with the | ||||
|
Juan C. Espinoza
|
r1326 | optional description parameter see the examples. | ||
| Parameters: | ||||
| ----------- | ||||
| path : str | ||||
| Path where files will be saved. | ||||
| blocksPerFile : int | ||||
| Number of blocks per file | ||||
| metadataList : list | ||||
| List of the dataOut attributes that will be saved as metadata | ||||
| dataList : int | ||||
| List of the dataOut attributes that will be saved as data | ||||
| setType : bool | ||||
| If True the name of the files corresponds to the timestamp of the data | ||||
| description : dict, optional | ||||
| Dictionary with the desired description of the HDF5 file | ||||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1326 | Examples | ||
| -------- | ||||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1326 | desc = { | ||
| 'data_output': {'winds': ['z', 'w', 'v']}, | ||||
| 'utctime': 'timestamps', | ||||
| 'heightList': 'heights' | ||||
| } | ||||
| desc = { | ||||
| 'data_output': ['z', 'w', 'v'], | ||||
| 'utctime': 'timestamps', | ||||
| 'heightList': 'heights' | ||||
| } | ||||
| desc = { | ||||
| 'Data': { | ||||
| 'data_output': 'winds', | ||||
| 'utctime': 'timestamps' | ||||
| }, | ||||
| 'Metadata': { | ||||
| 'heightList': 'heights' | ||||
| } | ||||
| } | ||||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1326 | writer = proc_unit.addOperation(name='HDFWriter') | ||
| writer.addParameter(name='path', value='/path/to/file') | ||||
| writer.addParameter(name='blocksPerFile', value='32') | ||||
| writer.addParameter(name='metadataList', value='heightList,timeZone') | ||||
| writer.addParameter(name='dataList',value='data_output,utctime') | ||||
| # writer.addParameter(name='description',value=json.dumps(desc)) | ||||
|
Juan C. Espinoza
|
r1232 | |||
|
Juan C. Espinoza
|
r1326 | """ | ||
|
Juan C. Espinoza
|
r1232 | |||
| r1596 | ext = ".hdf5" | |||
| optchar = "D" | ||||
| filename = None | ||||
| path = None | ||||
| setFile = None | ||||
| fp = None | ||||
| firsttime = True | ||||
|
Juan C. Espinoza
|
r1232 | #Configurations | ||
| blocksPerFile = None | ||||
| r1596 | blockIndex = None | |||
| dataOut = None | ||||
|
Juan C. Espinoza
|
r1232 | #Data Arrays | ||
| dataList = None | ||||
| metadataList = None | ||||
| r1596 | currentDay = None | |||
| lastTime = None | ||||
| last_Azipos = None | ||||
| last_Elepos = None | ||||
| mode = None | ||||
|
avaldezp
|
r1416 | #----------------------- | ||
| r1596 | Typename = None | |||
| mask = False | ||||
| setChannel = None | ||||
|
Juan C. Espinoza
|
r1232 | |||
| def __init__(self): | ||||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1232 | Operation.__init__(self) | ||
| return | ||||
|
avaldezp
|
r1416 | def set_kwargs(self, **kwargs): | ||
| for key, value in kwargs.items(): | ||||
| setattr(self, key, value) | ||||
| def set_kwargs_obj(self,obj, **kwargs): | ||||
| for key, value in kwargs.items(): | ||||
| setattr(obj, key, value) | ||||
| r1596 | def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None,type_data=None, localtime=True,setChannel=None, **kwargs): | |||
|
Juan C. Espinoza
|
r1232 | self.path = path | ||
| self.blocksPerFile = blocksPerFile | ||||
| r1596 | self.metadataList = metadataList | |||
| self.dataList = [s.strip() for s in dataList] | ||||
| self.setChannel = setChannel | ||||
| r1442 | self.setType = setType | |||
|
avaldezp
|
r1464 | if self.setType == "weather": | ||
|
avaldezp
|
r1416 | self.set_kwargs(**kwargs) | ||
| self.set_kwargs_obj(self.dataOut,**kwargs) | ||||
|
Juan C. Espinoza
|
r1514 | self.weather_vars = { | ||
| 'S' : 0, | ||||
| 'V' : 1, | ||||
| 'W' : 2, | ||||
| 'SNR' : 3, | ||||
| 'Z' : 4, | ||||
| 'D' : 5, | ||||
| 'P' : 6, | ||||
| 'R' : 7, | ||||
| } | ||||
| r1449 | ||||
|
Juan C. Espinoza
|
r1536 | if localtime: | ||
| self.getDateTime = datetime.datetime.fromtimestamp | ||||
| else: | ||||
| self.getDateTime = datetime.datetime.utcfromtimestamp | ||||
|
avaldezp
|
r1416 | |||
|
Juan C. Espinoza
|
r1326 | self.description = description | ||
|
avaldezp
|
r1416 | self.type_data=type_data | ||
|
Juan C. Espinoza
|
r1326 | |||
| r1339 | if self.metadataList is None: | |||
| self.metadataList = self.dataOut.metadata_list | ||||
|
Juan C. Espinoza
|
r1232 | |||
| dsList = [] | ||||
|
avaldezp
|
r1534 | |||
|
Juan C. Espinoza
|
r1232 | for i in range(len(self.dataList)): | ||
| dsDict = {} | ||||
| r1339 | if hasattr(self.dataOut, self.dataList[i]): | |||
| dataAux = getattr(self.dataOut, self.dataList[i]) | ||||
|
Juan C. Espinoza
|
r1514 | if self.setType == 'weather' and self.dataList[i] == 'data_param': | ||
| r1596 | if self.setChannel is None: | |||
| dataAux = dataAux[:,self.weather_vars[self.weather_var],:] | ||||
| else: | ||||
| dataAux = dataAux[self.setChannel,self.weather_vars[self.weather_var],:] | ||||
| dataAux = numpy.reshape(dataAux,(1,dataAux.shape[0],dataAux.shape[1])) | ||||
| r1339 | dsDict['variable'] = self.dataList[i] | |||
| else: | ||||
|
Juan C. Espinoza
|
r1514 | log.warning('Attribute {} not found in dataOut'.format(self.dataList[i]), self.name) | ||
| r1339 | continue | |||
|
Juan C. Espinoza
|
r1232 | |||
| if dataAux is None: | ||||
| continue | ||||
| r1726 | elif isinstance(dataAux, (int, float, numpy.integer, numpy.float_)): | |||
|
Juan C. Espinoza
|
r1232 | dsDict['nDim'] = 0 | ||
| else: | ||||
| dsDict['nDim'] = len(dataAux.shape) | ||||
| dsDict['shape'] = dataAux.shape | ||||
| dsDict['dsNumber'] = dataAux.shape[0] | ||||
|
Juan C. Espinoza
|
r1326 | dsDict['dtype'] = dataAux.dtype | ||
|
Juan C. Espinoza
|
r1232 | dsList.append(dsDict) | ||
| self.dsList = dsList | ||||
| self.currentDay = self.dataOut.datatime.date() | ||||
| def timeFlag(self): | ||||
| currentTime = self.dataOut.utctime | ||||
|
Juan C. Espinoza
|
r1536 | dt = self.getDateTime(currentTime) | ||
| dataDay = int(dt.strftime('%j')) | ||||
|
Juan C. Espinoza
|
r1232 | |||
| if self.lastTime is None: | ||||
| self.lastTime = currentTime | ||||
| self.currentDay = dataDay | ||||
| return False | ||||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1232 | timeDiff = currentTime - self.lastTime | ||
| #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora | ||||
| if dataDay != self.currentDay: | ||||
| self.currentDay = dataDay | ||||
| return True | ||||
| elif timeDiff > 3*60*60: | ||||
| self.lastTime = currentTime | ||||
| return True | ||||
| else: | ||||
| self.lastTime = currentTime | ||||
| return False | ||||
| r1339 | def run(self, dataOut, path, blocksPerFile=10, metadataList=None, | |||
|
Juan C. Espinoza
|
r1479 | dataList=[], setType=None, description={}, mode= None, | ||
|
Juan C. Espinoza
|
r1536 | type_data=None, Reset = False, localtime=True, **kwargs): | ||
| r1449 | ||||
| if Reset: | ||||
| self.isConfig = False | ||||
| self.closeFile() | ||||
| self.lastTime = None | ||||
| self.blockIndex = 0 | ||||
|
Juan C. Espinoza
|
r1232 | |||
| self.dataOut = dataOut | ||||
|
avaldezp
|
r1405 | self.mode = mode | ||
| r1459 | ||||
|
Juan C. Espinoza
|
r1232 | if not(self.isConfig): | ||
|
avaldezp
|
r1405 | self.setup(path=path, blocksPerFile=blocksPerFile, | ||
|
Juan C. Espinoza
|
r1232 | metadataList=metadataList, dataList=dataList, | ||
|
Juan C. Espinoza
|
r1536 | setType=setType, description=description,type_data=type_data, | ||
| localtime=localtime, **kwargs) | ||||
|
Juan C. Espinoza
|
r1232 | |||
| self.isConfig = True | ||||
| self.setNextFile() | ||||
| self.putData() | ||||
| return | ||||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1232 | def setNextFile(self): | ||
|
Juan C. Espinoza
|
r1479 | |||
|
Juan C. Espinoza
|
r1232 | ext = self.ext | ||
| path = self.path | ||||
| setFile = self.setFile | ||||
|
Juan C. Espinoza
|
r1536 | dt = self.getDateTime(self.dataOut.utctime) | ||
|
Juan C. Espinoza
|
r1479 | |||
| if self.setType == 'weather': | ||||
|
Juan C. Espinoza
|
r1536 | subfolder = dt.strftime('%Y-%m-%dT%H-00-00') | ||
| r1562 | subfolder = '' | |||
|
Juan C. Espinoza
|
r1479 | else: | ||
|
Juan C. Espinoza
|
r1536 | subfolder = dt.strftime('d%Y%j') | ||
|
Juan C. Espinoza
|
r1479 | |||
|
Juan C. Espinoza
|
r1232 | fullpath = os.path.join(path, subfolder) | ||
| if os.path.exists(fullpath): | ||||
| filesList = os.listdir(fullpath) | ||||
| filesList = [k for k in filesList if k.startswith(self.optchar)] | ||||
| if len( filesList ) > 0: | ||||
| filesList = sorted(filesList, key=str.lower) | ||||
| filen = filesList[-1] | ||||
| # el filename debera tener el siguiente formato | ||||
| # 0 1234 567 89A BCDE (hex) | ||||
| # x YYYY DDD SSS .ext | ||||
| if isNumber(filen[8:11]): | ||||
| setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file | ||||
| else: | ||||
| setFile = -1 | ||||
| else: | ||||
| setFile = -1 #inicializo mi contador de seteo | ||||
| else: | ||||
| os.makedirs(fullpath) | ||||
| setFile = -1 #inicializo mi contador de seteo | ||||
| if self.setType is None: | ||||
| setFile += 1 | ||||
| file = '%s%4.4d%3.3d%03d%s' % (self.optchar, | ||||
|
Juan C. Espinoza
|
r1536 | dt.year, | ||
| int(dt.strftime('%j')), | ||||
|
Juan C. Espinoza
|
r1232 | setFile, | ||
| ext ) | ||||
|
avaldezp
|
r1416 | elif self.setType == "weather": | ||
| r1466 | #SOPHY_20200505_140215_E10.0_Z.h5 | |||
| #SOPHY_20200505_140215_A40.0_Z.h5 | ||||
| r1459 | if self.dataOut.flagMode == 1: #'AZI' #PPI | |||
| r1562 | ang_type = 'EL' | |||
| mode_type = 'PPI' | ||||
| r1461 | len_aux = int(self.dataOut.data_ele.shape[0]/4) | |||
|
avaldezp
|
r1463 | mean = numpy.mean(self.dataOut.data_ele[len_aux:-len_aux]) | ||
| r1461 | ang_ = round(mean,1) | |||
| r1459 | elif self.dataOut.flagMode == 0: #'ELE' #RHI | |||
| r1562 | ang_type = 'AZ' | |||
| mode_type = 'RHI' | ||||
| r1461 | len_aux = int(self.dataOut.data_azi.shape[0]/4) | |||
|
avaldezp
|
r1463 | mean = numpy.mean(self.dataOut.data_azi[len_aux:-len_aux]) | ||
| r1461 | ang_ = round(mean,1) | |||
| r1459 | ||||
| r1466 | file = '%s_%2.2d%2.2d%2.2d_%2.2d%2.2d%2.2d_%s%2.1f_%s%s' % ( | |||
| 'SOPHY', | ||||
|
Juan C. Espinoza
|
r1536 | dt.year, | ||
| dt.month, | ||||
| dt.day, | ||||
| dt.hour, | ||||
| dt.minute, | ||||
| dt.second, | ||||
| r1562 | ang_type[0], | |||
| r1466 | ang_, | |||
|
Juan C. Espinoza
|
r1514 | self.weather_var, | ||
| r1459 | ext ) | |||
| r1562 | subfolder = '{}_{}_{}_{:2.1f}'.format(self.weather_var, mode_type, ang_type, ang_) | |||
| fullpath = os.path.join(path, subfolder) | ||||
| if not os.path.exists(fullpath): | ||||
| os.makedirs(fullpath) | ||||
|
Juan C. Espinoza
|
r1232 | else: | ||
|
Juan C. Espinoza
|
r1536 | setFile = dt.hour*60+dt.minute | ||
|
Juan C. Espinoza
|
r1232 | file = '%s%4.4d%3.3d%04d%s' % (self.optchar, | ||
|
Juan C. Espinoza
|
r1536 | dt.year, | ||
| int(dt.strftime('%j')), | ||||
|
Juan C. Espinoza
|
r1232 | setFile, | ||
| ext ) | ||||
| self.filename = os.path.join( path, subfolder, file ) | ||||
| self.fp = h5py.File(self.filename, 'w') | ||||
| #write metadata | ||||
| self.writeMetadata(self.fp) | ||||
| #Write data | ||||
| self.writeData(self.fp) | ||||
|
Juan C. Espinoza
|
r1326 | def getLabel(self, name, x=None): | ||
| if x is None: | ||||
| if 'Data' in self.description: | ||||
| data = self.description['Data'] | ||||
| if 'Metadata' in self.description: | ||||
| data.update(self.description['Metadata']) | ||||
| else: | ||||
| data = self.description | ||||
| if name in data: | ||||
| if isinstance(data[name], str): | ||||
| return data[name] | ||||
| elif isinstance(data[name], list): | ||||
| return None | ||||
| elif isinstance(data[name], dict): | ||||
| for key, value in data[name].items(): | ||||
| return key | ||||
| return name | ||||
| else: | ||||
|
Juan C. Espinoza
|
r1482 | if 'Data' in self.description: | ||
| data = self.description['Data'] | ||||
| if 'Metadata' in self.description: | ||||
| data.update(self.description['Metadata']) | ||||
|
Juan C. Espinoza
|
r1326 | else: | ||
|
Juan C. Espinoza
|
r1482 | data = self.description | ||
| if name in data: | ||||
| if isinstance(data[name], list): | ||||
| return data[name][x] | ||||
| elif isinstance(data[name], dict): | ||||
| for key, value in data[name].items(): | ||||
|
Juan C. Espinoza
|
r1326 | return value[x] | ||
| r1339 | if 'cspc' in name: | |||
| return 'pair{:02d}'.format(x) | ||||
| else: | ||||
| return 'channel{:02d}'.format(x) | ||||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1232 | def writeMetadata(self, fp): | ||
|
Juan C. Espinoza
|
r1326 | if self.description: | ||
| if 'Metadata' in self.description: | ||||
| grp = fp.create_group('Metadata') | ||||
| else: | ||||
| grp = fp | ||||
| else: | ||||
| grp = fp.create_group('Metadata') | ||||
|
Juan C. Espinoza
|
r1232 | |||
| for i in range(len(self.metadataList)): | ||||
| if not hasattr(self.dataOut, self.metadataList[i]): | ||||
| log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name) | ||||
| continue | ||||
| value = getattr(self.dataOut, self.metadataList[i]) | ||||
|
Juan C. Espinoza
|
r1326 | if isinstance(value, bool): | ||
| if value is True: | ||||
| value = 1 | ||||
| else: | ||||
| value = 0 | ||||
| grp.create_dataset(self.getLabel(self.metadataList[i]), data=value) | ||||
|
Juan C. Espinoza
|
r1232 | return | ||
| def writeData(self, fp): | ||||
| r1449 | ||||
|
Juan C. Espinoza
|
r1326 | if self.description: | ||
| if 'Data' in self.description: | ||||
| grp = fp.create_group('Data') | ||||
| else: | ||||
| grp = fp | ||||
| else: | ||||
| grp = fp.create_group('Data') | ||||
|
Juan C. Espinoza
|
r1232 | dtsets = [] | ||
| data = [] | ||||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1232 | for dsInfo in self.dsList: | ||
|
Juan C. Espinoza
|
r1482 | |||
|
Juan C. Espinoza
|
r1232 | if dsInfo['nDim'] == 0: | ||
| ds = grp.create_dataset( | ||||
|
avaldezp
|
r1405 | self.getLabel(dsInfo['variable']), | ||
|
Juan C. Espinoza
|
r1232 | (self.blocksPerFile, ), | ||
|
avaldezp
|
r1405 | chunks=True, | ||
|
Juan C. Espinoza
|
r1232 | dtype=numpy.float64) | ||
| dtsets.append(ds) | ||||
| data.append((dsInfo['variable'], -1)) | ||||
| else: | ||||
|
Juan C. Espinoza
|
r1326 | label = self.getLabel(dsInfo['variable']) | ||
| if label is not None: | ||||
| sgrp = grp.create_group(label) | ||||
| else: | ||||
| sgrp = grp | ||||
|
Juan C. Espinoza
|
r1482 | if self.blocksPerFile == 1: | ||
| shape = dsInfo['shape'][1:] | ||||
| else: | ||||
| shape = (self.blocksPerFile, ) + dsInfo['shape'][1:] | ||||
|
Juan C. Espinoza
|
r1232 | for i in range(dsInfo['dsNumber']): | ||
| r1596 | if dsInfo['dsNumber']==1: | |||
| if self.setChannel==1: | ||||
| i=1 | ||||
|
Juan C. Espinoza
|
r1232 | ds = sgrp.create_dataset( | ||
|
avaldezp
|
r1405 | self.getLabel(dsInfo['variable'], i), | ||
|
Juan C. Espinoza
|
r1482 | shape, | ||
|
Juan C. Espinoza
|
r1326 | chunks=True, | ||
|
Juan C. Espinoza
|
r1514 | dtype=dsInfo['dtype'], | ||
| compression='gzip', | ||||
| ) | ||||
|
Juan C. Espinoza
|
r1232 | dtsets.append(ds) | ||
| data.append((dsInfo['variable'], i)) | ||||
| fp.flush() | ||||
Juan C. Espinoza
|
r1235 | log.log('Creating file: {}'.format(fp.filename), self.name) | ||
|
avaldezp
|
r1405 | |||
|
Juan C. Espinoza
|
r1232 | self.ds = dtsets | ||
| self.data = data | ||||
| self.firsttime = True | ||||
| self.blockIndex = 0 | ||||
| return | ||||
| def putData(self): | ||||
|
avaldezp
|
r1534 | |||
|
Juan C. Espinoza
|
r1514 | if (self.blockIndex == self.blocksPerFile) or self.timeFlag(): | ||
|
Juan C. Espinoza
|
r1232 | self.closeFile() | ||
| self.setNextFile() | ||||
| for i, ds in enumerate(self.ds): | ||||
| attr, ch = self.data[i] | ||||
| if ch == -1: | ||||
| ds[self.blockIndex] = getattr(self.dataOut, attr) | ||||
| else: | ||||
|
Juan C. Espinoza
|
r1482 | if self.blocksPerFile == 1: | ||
|
Juan C. Espinoza
|
r1514 | mask = self.dataOut.data_param[:,3,:][ch] < self.mask | ||
| tmp = getattr(self.dataOut, attr)[:,self.weather_vars[self.weather_var],:][ch] | ||||
| if self.mask: | ||||
| tmp[mask] = numpy.nan | ||||
|
avaldezp
|
r1534 | ds[:] = tmp | ||
|
Juan C. Espinoza
|
r1482 | else: | ||
| ds[self.blockIndex] = getattr(self.dataOut, attr)[ch] | ||||
|
Juan C. Espinoza
|
r1232 | |||
| self.fp.flush() | ||||
| self.blockIndex += 1 | ||||
|
Juan C. Espinoza
|
r1235 | log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name) | ||
|
Juan C. Espinoza
|
r1232 | |||
| return | ||||
| def closeFile(self): | ||||
| if self.blockIndex != self.blocksPerFile: | ||||
| for ds in self.ds: | ||||
| ds.resize(self.blockIndex, axis=0) | ||||
| r1343 | if self.fp: | |||
| self.fp.flush() | ||||
| self.fp.close() | ||||
|
Juan C. Espinoza
|
r1232 | |||
| def close(self): | ||||
| self.closeFile() | ||||
