##// END OF EJS Templates
schain
RSS
Se comenta la linea 181, el atributo self.interval no se usa y genera error, en los scripts que utilicen el Reader de hdf5 jroIO_param.py, se sugiere en la unidad de lectura añadir el utcoffset=-18000 debido a que el reader trabaja en UTC, si se actualiza a esta version se puede hacer la seleccion de intervalos de tiempo en el startTime y endTime
Alexander Valdez -
r1720:2b23d7682e63
parent child
Show More
Side by Side Unified
Context file:
r1720:2b23d7682e63 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,650 +1,650
1 import os
1 import os
2 import time
2 import time
3 import datetime
3 import datetime
4
4
5 import numpy
5 import numpy
6 import h5py
6 import h5py
7
7
8 import schainpy.admin
8 import schainpy.admin
9 from schainpy.model.data.jrodata import *
9 from schainpy.model.data.jrodata import *
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 from schainpy.utils import log
12 from schainpy.utils import log
13
13
14
14
15 class HDFReader(Reader, ProcessingUnit):
15 class HDFReader(Reader, ProcessingUnit):
16 """Processing unit to read HDF5 format files
16 """Processing unit to read HDF5 format files
17
17
18 This unit reads HDF5 files created with `HDFWriter` operation contains
18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 attributes.
20 attributes.
21 It is possible to read any HDF5 file by given the structure in the `description`
21 It is possible to read any HDF5 file by given the structure in the `description`
22 parameter, also you can add extra values to metadata with the parameter `extras`.
22 parameter, also you can add extra values to metadata with the parameter `extras`.
23
23
24 Parameters:
24 Parameters:
25 -----------
25 -----------
26 path : str
26 path : str
27 Path where files are located.
27 Path where files are located.
28 startDate : date
28 startDate : date
29 Start date of the files
29 Start date of the files
30 endDate : list
30 endDate : list
31 End date of the files
31 End date of the files
32 startTime : time
32 startTime : time
33 Start time of the files
33 Start time of the files
34 endTime : time
34 endTime : time
35 End time of the files
35 End time of the files
36 description : dict, optional
36 description : dict, optional
37 Dictionary with the description of the HDF5 file
37 Dictionary with the description of the HDF5 file
38 extras : dict, optional
38 extras : dict, optional
39 Dictionary with extra metadata to be be added to `dataOut`
39 Dictionary with extra metadata to be be added to `dataOut`
40
40
41 Attention: Be carefull, add attribute utcoffset, in the last part of reader in order to work in Local Time without time problems.
41 Attention: Be carefull, add attribute utcoffset, in the last part of reader in order to work in Local Time without time problems.
42
42
43 -----------
43 -----------
44 utcoffset='-18000'
44 utcoffset='-18000'
45
45
46
46
47 Examples
47 Examples
48 --------
48 --------
49
49
50 desc = {
50 desc = {
51 'Data': {
51 'Data': {
52 'data_output': ['u', 'v', 'w'],
52 'data_output': ['u', 'v', 'w'],
53 'utctime': 'timestamps',
53 'utctime': 'timestamps',
54 } ,
54 } ,
55 'Metadata': {
55 'Metadata': {
56 'heightList': 'heights'
56 'heightList': 'heights'
57 }
57 }
58 }
58 }
59
59
60 desc = {
60 desc = {
61 'Data': {
61 'Data': {
62 'data_output': 'winds',
62 'data_output': 'winds',
63 'utctime': 'timestamps'
63 'utctime': 'timestamps'
64 },
64 },
65 'Metadata': {
65 'Metadata': {
66 'heightList': 'heights'
66 'heightList': 'heights'
67 }
67 }
68 }
68 }
69
69
70 extras = {
70 extras = {
71 'timeZone': 300
71 'timeZone': 300
72 }
72 }
73
73
74 reader = project.addReadUnit(
74 reader = project.addReadUnit(
75 name='HDFReader',
75 name='HDFReader',
76 path='/path/to/files',
76 path='/path/to/files',
77 startDate='2019/01/01',
77 startDate='2019/01/01',
78 endDate='2019/01/31',
78 endDate='2019/01/31',
79 startTime='00:00:00',
79 startTime='00:00:00',
80 endTime='23:59:59',
80 endTime='23:59:59',
81 utcoffset='-18000'
81 utcoffset='-18000'
82 # description=json.dumps(desc),
82 # description=json.dumps(desc),
83 # extras=json.dumps(extras),
83 # extras=json.dumps(extras),
84 )
84 )
85
85
86 """
86 """
87
87
88 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
88 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
89
89
90 def __init__(self):
90 def __init__(self):
91 ProcessingUnit.__init__(self)
91 ProcessingUnit.__init__(self)
92 self.dataOut = Parameters()
92 self.dataOut = Parameters()
93 self.ext = ".hdf5"
93 self.ext = ".hdf5"
94 self.optchar = "D"
94 self.optchar = "D"
95 self.meta = {}
95 self.meta = {}
96 self.data = {}
96 self.data = {}
97 self.open_file = h5py.File
97 self.open_file = h5py.File
98 self.open_mode = 'r'
98 self.open_mode = 'r'
99 self.description = {}
99 self.description = {}
100 self.extras = {}
100 self.extras = {}
101 self.filefmt = "*%Y%j***"
101 self.filefmt = "*%Y%j***"
102 self.folderfmt = "*%Y%j"
102 self.folderfmt = "*%Y%j"
103 self.utcoffset = 0
103 self.utcoffset = 0
104
104
105 def setup(self, **kwargs):
105 def setup(self, **kwargs):
106
106
107 self.set_kwargs(**kwargs)
107 self.set_kwargs(**kwargs)
108 if not self.ext.startswith('.'):
108 if not self.ext.startswith('.'):
109 self.ext = '.{}'.format(self.ext)
109 self.ext = '.{}'.format(self.ext)
110
110
111 if self.online:
111 if self.online:
112 log.log("Searching files in online mode...", self.name)
112 log.log("Searching files in online mode...", self.name)
113
113
114 for nTries in range(self.nTries):
114 for nTries in range(self.nTries):
115 fullpath = self.searchFilesOnLine(self.path, self.startDate,
115 fullpath = self.searchFilesOnLine(self.path, self.startDate,
116 self.endDate, self.expLabel, self.ext, self.walk,
116 self.endDate, self.expLabel, self.ext, self.walk,
117 self.filefmt, self.folderfmt)
117 self.filefmt, self.folderfmt)
118 try:
118 try:
119 fullpath = next(fullpath)
119 fullpath = next(fullpath)
120 except:
120 except:
121 fullpath = None
121 fullpath = None
122
122
123 if fullpath:
123 if fullpath:
124 break
124 break
125
125
126 log.warning(
126 log.warning(
127 'Waiting {} sec for a valid file in {}: try {} ...'.format(
127 'Waiting {} sec for a valid file in {}: try {} ...'.format(
128 self.delay, self.path, nTries + 1),
128 self.delay, self.path, nTries + 1),
129 self.name)
129 self.name)
130 time.sleep(self.delay)
130 time.sleep(self.delay)
131
131
132 if not(fullpath):
132 if not(fullpath):
133 raise schainpy.admin.SchainError(
133 raise schainpy.admin.SchainError(
134 'There isn\'t any valid file in {}'.format(self.path))
134 'There isn\'t any valid file in {}'.format(self.path))
135
135
136 pathname, filename = os.path.split(fullpath)
136 pathname, filename = os.path.split(fullpath)
137 self.year = int(filename[1:5])
137 self.year = int(filename[1:5])
138 self.doy = int(filename[5:8])
138 self.doy = int(filename[5:8])
139 self.set = int(filename[8:11]) - 1
139 self.set = int(filename[8:11]) - 1
140 else:
140 else:
141 log.log("Searching files in {}".format(self.path), self.name)
141 log.log("Searching files in {}".format(self.path), self.name)
142 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
142 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
143 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
143 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
144
144
145 self.setNextFile()
145 self.setNextFile()
146
146
147 return
147 return
148
148
149 def readFirstHeader(self):
149 def readFirstHeader(self):
150 '''Read metadata and data'''
150 '''Read metadata and data'''
151
151
152 self.__readMetadata()
152 self.__readMetadata()
153 self.__readData()
153 self.__readData()
154 self.__setBlockList()
154 self.__setBlockList()
155
155
156 if 'type' in self.meta:
156 if 'type' in self.meta:
157 self.dataOut = eval(self.meta['type'])()
157 self.dataOut = eval(self.meta['type'])()
158
158
159 for attr in self.meta:
159 for attr in self.meta:
160 setattr(self.dataOut, attr, self.meta[attr])
160 setattr(self.dataOut, attr, self.meta[attr])
161
161
162 self.blockIndex = 0
162 self.blockIndex = 0
163
163
164 return
164 return
165
165
166 def __setBlockList(self):
166 def __setBlockList(self):
167 '''
167 '''
168 Selects the data within the times defined
168 Selects the data within the times defined
169
169
170 self.fp
170 self.fp
171 self.startTime
171 self.startTime
172 self.endTime
172 self.endTime
173 self.blockList
173 self.blockList
174 self.blocksPerFile
174 self.blocksPerFile
175
175
176 '''
176 '''
177
177
178 startTime = self.startTime
178 startTime = self.startTime
179 endTime = self.endTime
179 endTime = self.endTime
180 thisUtcTime = self.data['utctime'] + self.utcoffset
180 thisUtcTime = self.data['utctime'] + self.utcoffset
181 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
181 #self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
182 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
182 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
183
183
184 thisDate = thisDatetime.date()
184 thisDate = thisDatetime.date()
185 thisTime = thisDatetime.time()
185 thisTime = thisDatetime.time()
186 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
186 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
187 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
187 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
188 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
188 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
189
189
190 self.blockList = ind
190 self.blockList = ind
191 self.blocksPerFile = len(ind)
191 self.blocksPerFile = len(ind)
192
192
193 if len(ind)==0:
193 if len(ind)==0:
194 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.blockIndex,
194 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.blockIndex,
195 self.blocksPerFile,
195 self.blocksPerFile,
196 thisDatetime))
196 thisDatetime))
197 self.setNextFile()
197 self.setNextFile()
198
198
199 return
199 return
200
200
201 def __readMetadata(self):
201 def __readMetadata(self):
202 '''
202 '''
203 Reads Metadata
203 Reads Metadata
204 '''
204 '''
205
205
206 meta = {}
206 meta = {}
207
207
208 if self.description:
208 if self.description:
209 for key, value in self.description['Metadata'].items():
209 for key, value in self.description['Metadata'].items():
210 meta[key] = self.fp[value][()]
210 meta[key] = self.fp[value][()]
211 else:
211 else:
212 grp = self.fp['Metadata']
212 grp = self.fp['Metadata']
213 for name in grp:
213 for name in grp:
214 meta[name] = grp[name][()]
214 meta[name] = grp[name][()]
215
215
216 if self.extras:
216 if self.extras:
217 for key, value in self.extras.items():
217 for key, value in self.extras.items():
218 meta[key] = value
218 meta[key] = value
219 self.meta = meta
219 self.meta = meta
220
220
221 return
221 return
222
222
223 def __readData(self):
223 def __readData(self):
224
224
225 data = {}
225 data = {}
226
226
227 if self.description:
227 if self.description:
228 for key, value in self.description['Data'].items():
228 for key, value in self.description['Data'].items():
229 if isinstance(value, str):
229 if isinstance(value, str):
230 if isinstance(self.fp[value], h5py.Dataset):
230 if isinstance(self.fp[value], h5py.Dataset):
231 data[key] = self.fp[value][()]
231 data[key] = self.fp[value][()]
232 elif isinstance(self.fp[value], h5py.Group):
232 elif isinstance(self.fp[value], h5py.Group):
233 array = []
233 array = []
234 for ch in self.fp[value]:
234 for ch in self.fp[value]:
235 array.append(self.fp[value][ch][()])
235 array.append(self.fp[value][ch][()])
236 data[key] = numpy.array(array)
236 data[key] = numpy.array(array)
237 elif isinstance(value, list):
237 elif isinstance(value, list):
238 array = []
238 array = []
239 for ch in value:
239 for ch in value:
240 array.append(self.fp[ch][()])
240 array.append(self.fp[ch][()])
241 data[key] = numpy.array(array)
241 data[key] = numpy.array(array)
242 else:
242 else:
243 grp = self.fp['Data']
243 grp = self.fp['Data']
244 for name in grp:
244 for name in grp:
245 if isinstance(grp[name], h5py.Dataset):
245 if isinstance(grp[name], h5py.Dataset):
246 array = grp[name][()]
246 array = grp[name][()]
247 elif isinstance(grp[name], h5py.Group):
247 elif isinstance(grp[name], h5py.Group):
248 array = []
248 array = []
249 for ch in grp[name]:
249 for ch in grp[name]:
250 array.append(grp[name][ch][()])
250 array.append(grp[name][ch][()])
251 array = numpy.array(array)
251 array = numpy.array(array)
252 else:
252 else:
253 log.warning('Unknown type: {}'.format(name))
253 log.warning('Unknown type: {}'.format(name))
254
254
255 if name in self.description:
255 if name in self.description:
256 key = self.description[name]
256 key = self.description[name]
257 else:
257 else:
258 key = name
258 key = name
259 data[key] = array
259 data[key] = array
260
260
261 self.data = data
261 self.data = data
262 return
262 return
263
263
264 def getData(self):
264 def getData(self):
265
265
266 for attr in self.data:
266 for attr in self.data:
267 if self.data[attr].ndim == 1:
267 if self.data[attr].ndim == 1:
268 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
268 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
269 else:
269 else:
270 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
270 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
271
271
272 self.dataOut.flagNoData = False
272 self.dataOut.flagNoData = False
273 self.blockIndex += 1
273 self.blockIndex += 1
274
274
275 log.log("Block No. {}/{} -> {}".format(
275 log.log("Block No. {}/{} -> {}".format(
276 self.blockIndex,
276 self.blockIndex,
277 self.blocksPerFile,
277 self.blocksPerFile,
278 self.dataOut.datatime.ctime()), self.name)
278 self.dataOut.datatime.ctime()), self.name)
279
279
280 return
280 return
281
281
282 def run(self, **kwargs):
282 def run(self, **kwargs):
283
283
284 if not(self.isConfig):
284 if not(self.isConfig):
285 self.setup(**kwargs)
285 self.setup(**kwargs)
286 self.isConfig = True
286 self.isConfig = True
287
287
288 if self.blockIndex == self.blocksPerFile:
288 if self.blockIndex == self.blocksPerFile:
289 self.setNextFile()
289 self.setNextFile()
290
290
291 self.getData()
291 self.getData()
292
292
293 return
293 return
294
294
295 @MPDecorator
295 @MPDecorator
296 class HDFWriter(Operation):
296 class HDFWriter(Operation):
297 """Operation to write HDF5 files.
297 """Operation to write HDF5 files.
298
298
299 The HDF5 file contains by default two groups Data and Metadata where
299 The HDF5 file contains by default two groups Data and Metadata where
300 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
300 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
301 parameters, data attributes are normaly time dependent where the metadata
301 parameters, data attributes are normaly time dependent where the metadata
302 are not.
302 are not.
303 It is possible to customize the structure of the HDF5 file with the
303 It is possible to customize the structure of the HDF5 file with the
304 optional description parameter see the examples.
304 optional description parameter see the examples.
305
305
306 Parameters:
306 Parameters:
307 -----------
307 -----------
308 path : str
308 path : str
309 Path where files will be saved.
309 Path where files will be saved.
310 blocksPerFile : int
310 blocksPerFile : int
311 Number of blocks per file
311 Number of blocks per file
312 metadataList : list
312 metadataList : list
313 List of the dataOut attributes that will be saved as metadata
313 List of the dataOut attributes that will be saved as metadata
314 dataList : int
314 dataList : int
315 List of the dataOut attributes that will be saved as data
315 List of the dataOut attributes that will be saved as data
316 setType : bool
316 setType : bool
317 If True the name of the files corresponds to the timestamp of the data
317 If True the name of the files corresponds to the timestamp of the data
318 description : dict, optional
318 description : dict, optional
319 Dictionary with the desired description of the HDF5 file
319 Dictionary with the desired description of the HDF5 file
320
320
321 Examples
321 Examples
322 --------
322 --------
323
323
324 desc = {
324 desc = {
325 'data_output': {'winds': ['z', 'w', 'v']},
325 'data_output': {'winds': ['z', 'w', 'v']},
326 'utctime': 'timestamps',
326 'utctime': 'timestamps',
327 'heightList': 'heights'
327 'heightList': 'heights'
328 }
328 }
329 desc = {
329 desc = {
330 'data_output': ['z', 'w', 'v'],
330 'data_output': ['z', 'w', 'v'],
331 'utctime': 'timestamps',
331 'utctime': 'timestamps',
332 'heightList': 'heights'
332 'heightList': 'heights'
333 }
333 }
334 desc = {
334 desc = {
335 'Data': {
335 'Data': {
336 'data_output': 'winds',
336 'data_output': 'winds',
337 'utctime': 'timestamps'
337 'utctime': 'timestamps'
338 },
338 },
339 'Metadata': {
339 'Metadata': {
340 'heightList': 'heights'
340 'heightList': 'heights'
341 }
341 }
342 }
342 }
343
343
344 writer = proc_unit.addOperation(name='HDFWriter')
344 writer = proc_unit.addOperation(name='HDFWriter')
345 writer.addParameter(name='path', value='/path/to/file')
345 writer.addParameter(name='path', value='/path/to/file')
346 writer.addParameter(name='blocksPerFile', value='32')
346 writer.addParameter(name='blocksPerFile', value='32')
347 writer.addParameter(name='metadataList', value='heightList,timeZone')
347 writer.addParameter(name='metadataList', value='heightList,timeZone')
348 writer.addParameter(name='dataList',value='data_output,utctime')
348 writer.addParameter(name='dataList',value='data_output,utctime')
349 # writer.addParameter(name='description',value=json.dumps(desc))
349 # writer.addParameter(name='description',value=json.dumps(desc))
350
350
351 """
351 """
352
352
353 ext = ".hdf5"
353 ext = ".hdf5"
354 optchar = "D"
354 optchar = "D"
355 filename = None
355 filename = None
356 path = None
356 path = None
357 setFile = None
357 setFile = None
358 fp = None
358 fp = None
359 firsttime = True
359 firsttime = True
360 #Configurations
360 #Configurations
361 blocksPerFile = None
361 blocksPerFile = None
362 blockIndex = None
362 blockIndex = None
363 dataOut = None
363 dataOut = None
364 #Data Arrays
364 #Data Arrays
365 dataList = None
365 dataList = None
366 metadataList = None
366 metadataList = None
367 currentDay = None
367 currentDay = None
368 lastTime = None
368 lastTime = None
369
369
370 def __init__(self):
370 def __init__(self):
371
371
372 Operation.__init__(self)
372 Operation.__init__(self)
373 return
373 return
374
374
375 def set_kwargs(self, **kwargs):
375 def set_kwargs(self, **kwargs):
376
376
377 for key, value in kwargs.items():
377 for key, value in kwargs.items():
378 setattr(self, key, value)
378 setattr(self, key, value)
379
379
380 def set_kwargs_obj(self, obj, **kwargs):
380 def set_kwargs_obj(self, obj, **kwargs):
381
381
382 for key, value in kwargs.items():
382 for key, value in kwargs.items():
383 setattr(obj, key, value)
383 setattr(obj, key, value)
384
384
385 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None, **kwargs):
385 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None, **kwargs):
386 self.path = path
386 self.path = path
387 self.blocksPerFile = blocksPerFile
387 self.blocksPerFile = blocksPerFile
388 self.metadataList = metadataList
388 self.metadataList = metadataList
389 self.dataList = [s.strip() for s in dataList]
389 self.dataList = [s.strip() for s in dataList]
390 self.setType = setType
390 self.setType = setType
391 self.description = description
391 self.description = description
392 self.set_kwargs(**kwargs)
392 self.set_kwargs(**kwargs)
393
393
394 if self.metadataList is None:
394 if self.metadataList is None:
395 self.metadataList = self.dataOut.metadata_list
395 self.metadataList = self.dataOut.metadata_list
396
396
397 tableList = []
397 tableList = []
398 dsList = []
398 dsList = []
399
399
400 for i in range(len(self.dataList)):
400 for i in range(len(self.dataList)):
401 dsDict = {}
401 dsDict = {}
402 if hasattr(self.dataOut, self.dataList[i]):
402 if hasattr(self.dataOut, self.dataList[i]):
403 dataAux = getattr(self.dataOut, self.dataList[i])
403 dataAux = getattr(self.dataOut, self.dataList[i])
404 dsDict['variable'] = self.dataList[i]
404 dsDict['variable'] = self.dataList[i]
405 else:
405 else:
406 log.warning('Attribute {} not found in dataOut', self.name)
406 log.warning('Attribute {} not found in dataOut', self.name)
407 continue
407 continue
408
408
409 if dataAux is None:
409 if dataAux is None:
410 continue
410 continue
411 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
411 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
412 dsDict['nDim'] = 0
412 dsDict['nDim'] = 0
413 else:
413 else:
414 dsDict['nDim'] = len(dataAux.shape)
414 dsDict['nDim'] = len(dataAux.shape)
415 dsDict['shape'] = dataAux.shape
415 dsDict['shape'] = dataAux.shape
416 dsDict['dsNumber'] = dataAux.shape[0]
416 dsDict['dsNumber'] = dataAux.shape[0]
417 dsDict['dtype'] = dataAux.dtype
417 dsDict['dtype'] = dataAux.dtype
418
418
419 dsList.append(dsDict)
419 dsList.append(dsDict)
420
420
421 self.dsList = dsList
421 self.dsList = dsList
422 self.currentDay = self.dataOut.datatime.date()
422 self.currentDay = self.dataOut.datatime.date()
423
423
424 def timeFlag(self):
424 def timeFlag(self):
425 currentTime = self.dataOut.utctime
425 currentTime = self.dataOut.utctime
426 timeTuple = time.localtime(currentTime)
426 timeTuple = time.localtime(currentTime)
427 dataDay = timeTuple.tm_yday
427 dataDay = timeTuple.tm_yday
428
428
429 if self.lastTime is None:
429 if self.lastTime is None:
430 self.lastTime = currentTime
430 self.lastTime = currentTime
431 self.currentDay = dataDay
431 self.currentDay = dataDay
432 return False
432 return False
433
433
434 timeDiff = currentTime - self.lastTime
434 timeDiff = currentTime - self.lastTime
435
435
436 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
436 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
437 if dataDay != self.currentDay:
437 if dataDay != self.currentDay:
438 self.currentDay = dataDay
438 self.currentDay = dataDay
439 return True
439 return True
440 elif timeDiff > 3*60*60:
440 elif timeDiff > 3*60*60:
441 self.lastTime = currentTime
441 self.lastTime = currentTime
442 return True
442 return True
443 else:
443 else:
444 self.lastTime = currentTime
444 self.lastTime = currentTime
445 return False
445 return False
446
446
447 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
447 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
448 dataList=[], setType=None, description={}, **kwargs):
448 dataList=[], setType=None, description={}, **kwargs):
449
449
450 self.dataOut = dataOut
450 self.dataOut = dataOut
451 self.set_kwargs_obj(self.dataOut, **kwargs)
451 self.set_kwargs_obj(self.dataOut, **kwargs)
452 if not(self.isConfig):
452 if not(self.isConfig):
453 self.setup(path=path, blocksPerFile=blocksPerFile,
453 self.setup(path=path, blocksPerFile=blocksPerFile,
454 metadataList=metadataList, dataList=dataList,
454 metadataList=metadataList, dataList=dataList,
455 setType=setType, description=description, **kwargs)
455 setType=setType, description=description, **kwargs)
456
456
457 self.isConfig = True
457 self.isConfig = True
458 self.setNextFile()
458 self.setNextFile()
459
459
460 self.putData()
460 self.putData()
461 return
461 return
462
462
463 def setNextFile(self):
463 def setNextFile(self):
464
464
465 ext = self.ext
465 ext = self.ext
466 path = self.path
466 path = self.path
467 setFile = self.setFile
467 setFile = self.setFile
468
468
469 timeTuple = time.localtime(self.dataOut.utctime)
469 timeTuple = time.localtime(self.dataOut.utctime)
470 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
470 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
471 fullpath = os.path.join(path, subfolder)
471 fullpath = os.path.join(path, subfolder)
472
472
473 if os.path.exists(fullpath):
473 if os.path.exists(fullpath):
474 filesList = os.listdir(fullpath)
474 filesList = os.listdir(fullpath)
475 filesList = [k for k in filesList if k.startswith(self.optchar)]
475 filesList = [k for k in filesList if k.startswith(self.optchar)]
476 if len(filesList) > 0:
476 if len(filesList) > 0:
477 filesList = sorted(filesList, key=str.lower)
477 filesList = sorted(filesList, key=str.lower)
478 filen = filesList[-1]
478 filen = filesList[-1]
479 # el filename debera tener el siguiente formato
479 # el filename debera tener el siguiente formato
480 # 0 1234 567 89A BCDE (hex)
480 # 0 1234 567 89A BCDE (hex)
481 # x YYYY DDD SSS .ext
481 # x YYYY DDD SSS .ext
482 if isNumber(filen[8:11]):
482 if isNumber(filen[8:11]):
483 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
483 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
484 else:
484 else:
485 setFile = -1
485 setFile = -1
486 else:
486 else:
487 setFile = -1 #inicializo mi contador de seteo
487 setFile = -1 #inicializo mi contador de seteo
488 else:
488 else:
489 os.makedirs(fullpath)
489 os.makedirs(fullpath)
490 setFile = -1 #inicializo mi contador de seteo
490 setFile = -1 #inicializo mi contador de seteo
491
491
492 if self.setType is None:
492 if self.setType is None:
493 setFile += 1
493 setFile += 1
494 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
494 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
495 timeTuple.tm_year,
495 timeTuple.tm_year,
496 timeTuple.tm_yday,
496 timeTuple.tm_yday,
497 setFile,
497 setFile,
498 ext)
498 ext)
499 else:
499 else:
500 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
500 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
501 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
501 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
502 timeTuple.tm_year,
502 timeTuple.tm_year,
503 timeTuple.tm_yday,
503 timeTuple.tm_yday,
504 setFile,
504 setFile,
505 ext)
505 ext)
506
506
507 self.filename = os.path.join(path, subfolder, file)
507 self.filename = os.path.join(path, subfolder, file)
508
508
509 #Setting HDF5 File
509 #Setting HDF5 File
510 self.fp = h5py.File(self.filename, 'w')
510 self.fp = h5py.File(self.filename, 'w')
511 #write metadata
511 #write metadata
512 self.writeMetadata(self.fp)
512 self.writeMetadata(self.fp)
513 #Write data
513 #Write data
514 self.writeData(self.fp)
514 self.writeData(self.fp)
515
515
516 def getLabel(self, name, x=None):
516 def getLabel(self, name, x=None):
517
517
518 if x is None:
518 if x is None:
519 if 'Data' in self.description:
519 if 'Data' in self.description:
520 data = self.description['Data']
520 data = self.description['Data']
521 if 'Metadata' in self.description:
521 if 'Metadata' in self.description:
522 data.update(self.description['Metadata'])
522 data.update(self.description['Metadata'])
523 else:
523 else:
524 data = self.description
524 data = self.description
525 if name in data:
525 if name in data:
526 if isinstance(data[name], str):
526 if isinstance(data[name], str):
527 return data[name]
527 return data[name]
528 elif isinstance(data[name], list):
528 elif isinstance(data[name], list):
529 return None
529 return None
530 elif isinstance(data[name], dict):
530 elif isinstance(data[name], dict):
531 for key, value in data[name].items():
531 for key, value in data[name].items():
532 return key
532 return key
533 return name
533 return name
534 else:
534 else:
535 if 'Metadata' in self.description:
535 if 'Metadata' in self.description:
536 meta = self.description['Metadata']
536 meta = self.description['Metadata']
537 else:
537 else:
538 meta = self.description
538 meta = self.description
539 if name in meta:
539 if name in meta:
540 if isinstance(meta[name], list):
540 if isinstance(meta[name], list):
541 return meta[name][x]
541 return meta[name][x]
542 elif isinstance(meta[name], dict):
542 elif isinstance(meta[name], dict):
543 for key, value in meta[name].items():
543 for key, value in meta[name].items():
544 return value[x]
544 return value[x]
545 if 'cspc' in name:
545 if 'cspc' in name:
546 return 'pair{:02d}'.format(x)
546 return 'pair{:02d}'.format(x)
547 else:
547 else:
548 return 'channel{:02d}'.format(x)
548 return 'channel{:02d}'.format(x)
549
549
550 def writeMetadata(self, fp):
550 def writeMetadata(self, fp):
551
551
552 if self.description:
552 if self.description:
553 if 'Metadata' in self.description:
553 if 'Metadata' in self.description:
554 grp = fp.create_group('Metadata')
554 grp = fp.create_group('Metadata')
555 else:
555 else:
556 grp = fp
556 grp = fp
557 else:
557 else:
558 grp = fp.create_group('Metadata')
558 grp = fp.create_group('Metadata')
559
559
560 for i in range(len(self.metadataList)):
560 for i in range(len(self.metadataList)):
561 if not hasattr(self.dataOut, self.metadataList[i]):
561 if not hasattr(self.dataOut, self.metadataList[i]):
562 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
562 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
563 continue
563 continue
564 value = getattr(self.dataOut, self.metadataList[i])
564 value = getattr(self.dataOut, self.metadataList[i])
565 if isinstance(value, bool):
565 if isinstance(value, bool):
566 if value is True:
566 if value is True:
567 value = 1
567 value = 1
568 else:
568 else:
569 value = 0
569 value = 0
570 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
570 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
571 return
571 return
572
572
573 def writeData(self, fp):
573 def writeData(self, fp):
574
574
575 if self.description:
575 if self.description:
576 if 'Data' in self.description:
576 if 'Data' in self.description:
577 grp = fp.create_group('Data')
577 grp = fp.create_group('Data')
578 else:
578 else:
579 grp = fp
579 grp = fp
580 else:
580 else:
581 grp = fp.create_group('Data')
581 grp = fp.create_group('Data')
582
582
583 dtsets = []
583 dtsets = []
584 data = []
584 data = []
585
585
586 for dsInfo in self.dsList:
586 for dsInfo in self.dsList:
587 if dsInfo['nDim'] == 0:
587 if dsInfo['nDim'] == 0:
588 ds = grp.create_dataset(
588 ds = grp.create_dataset(
589 self.getLabel(dsInfo['variable']),
589 self.getLabel(dsInfo['variable']),
590 (self.blocksPerFile,),
590 (self.blocksPerFile,),
591 chunks=True,
591 chunks=True,
592 dtype=numpy.float64)
592 dtype=numpy.float64)
593 dtsets.append(ds)
593 dtsets.append(ds)
594 data.append((dsInfo['variable'], -1))
594 data.append((dsInfo['variable'], -1))
595 else:
595 else:
596 label = self.getLabel(dsInfo['variable'])
596 label = self.getLabel(dsInfo['variable'])
597 if label is not None:
597 if label is not None:
598 sgrp = grp.create_group(label)
598 sgrp = grp.create_group(label)
599 else:
599 else:
600 sgrp = grp
600 sgrp = grp
601 for i in range(dsInfo['dsNumber']):
601 for i in range(dsInfo['dsNumber']):
602 ds = sgrp.create_dataset(
602 ds = sgrp.create_dataset(
603 self.getLabel(dsInfo['variable'], i),
603 self.getLabel(dsInfo['variable'], i),
604 (self.blocksPerFile,) + dsInfo['shape'][1:],
604 (self.blocksPerFile,) + dsInfo['shape'][1:],
605 chunks=True,
605 chunks=True,
606 dtype=dsInfo['dtype'])
606 dtype=dsInfo['dtype'])
607 dtsets.append(ds)
607 dtsets.append(ds)
608 data.append((dsInfo['variable'], i))
608 data.append((dsInfo['variable'], i))
609 fp.flush()
609 fp.flush()
610
610
611 log.log('Creating file: {}'.format(fp.filename), self.name)
611 log.log('Creating file: {}'.format(fp.filename), self.name)
612
612
613 self.ds = dtsets
613 self.ds = dtsets
614 self.data = data
614 self.data = data
615 self.firsttime = True
615 self.firsttime = True
616 self.blockIndex = 0
616 self.blockIndex = 0
617 return
617 return
618
618
619 def putData(self):
619 def putData(self):
620
620
621 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
621 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
622 self.closeFile()
622 self.closeFile()
623 self.setNextFile()
623 self.setNextFile()
624
624
625 for i, ds in enumerate(self.ds):
625 for i, ds in enumerate(self.ds):
626 attr, ch = self.data[i]
626 attr, ch = self.data[i]
627 if ch == -1:
627 if ch == -1:
628 ds[self.blockIndex] = getattr(self.dataOut, attr)
628 ds[self.blockIndex] = getattr(self.dataOut, attr)
629 else:
629 else:
630 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
630 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
631
631
632 self.fp.flush()
632 self.fp.flush()
633 self.blockIndex += 1
633 self.blockIndex += 1
634 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
634 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
635
635
636 return
636 return
637
637
638 def closeFile(self):
638 def closeFile(self):
639
639
640 if self.blockIndex != self.blocksPerFile:
640 if self.blockIndex != self.blocksPerFile:
641 for ds in self.ds:
641 for ds in self.ds:
642 ds.resize(self.blockIndex, axis=0)
642 ds.resize(self.blockIndex, axis=0)
643
643
644 if self.fp:
644 if self.fp:
645 self.fp.flush()
645 self.fp.flush()
646 self.fp.close()
646 self.fp.close()
647
647
648 def close(self):
648 def close(self):
649
649
650 self.closeFile()
650 self.closeFile()
General Comments 0
You need to be logged in to leave comments. Login now