##// END OF EJS Templates
schain
RSS
fix error of selection time in HDFReader
Alexander Valdez -
r1691:5adc7ae610db
parent child
Show More
Side by Side Unified
Context file:
r1691:5adc7ae610db Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,628 +1,638
1 import os
1 import os
2 import time
2 import time
3 import datetime
3 import datetime
4
4
5 import numpy
5 import numpy
6 import h5py
6 import h5py
7
7
8 import schainpy.admin
8 import schainpy.admin
9 from schainpy.model.data.jrodata import *
9 from schainpy.model.data.jrodata import *
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 from schainpy.utils import log
12 from schainpy.utils import log
13
13
14
14
15 class HDFReader(Reader, ProcessingUnit):
15 class HDFReader(Reader, ProcessingUnit):
16 """Processing unit to read HDF5 format files
16 """Processing unit to read HDF5 format files
17
17
18 This unit reads HDF5 files created with `HDFWriter` operation contains
18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 attributes.
20 attributes.
21 It is possible to read any HDF5 file by given the structure in the `description`
21 It is possible to read any HDF5 file by given the structure in the `description`
22 parameter, also you can add extra values to metadata with the parameter `extras`.
22 parameter, also you can add extra values to metadata with the parameter `extras`.
23
23
24 Parameters:
24 Parameters:
25 -----------
25 -----------
26 path : str
26 path : str
27 Path where files are located.
27 Path where files are located.
28 startDate : date
28 startDate : date
29 Start date of the files
29 Start date of the files
30 endDate : list
30 endDate : list
31 End date of the files
31 End date of the files
32 startTime : time
32 startTime : time
33 Start time of the files
33 Start time of the files
34 endTime : time
34 endTime : time
35 End time of the files
35 End time of the files
36 description : dict, optional
36 description : dict, optional
37 Dictionary with the description of the HDF5 file
37 Dictionary with the description of the HDF5 file
38 extras : dict, optional
38 extras : dict, optional
39 Dictionary with extra metadata to be be added to `dataOut`
39 Dictionary with extra metadata to be be added to `dataOut`
40
40
41 Examples
41 Examples
42 --------
42 --------
43
43
44 desc = {
44 desc = {
45 'Data': {
45 'Data': {
46 'data_output': ['u', 'v', 'w'],
46 'data_output': ['u', 'v', 'w'],
47 'utctime': 'timestamps',
47 'utctime': 'timestamps',
48 } ,
48 } ,
49 'Metadata': {
49 'Metadata': {
50 'heightList': 'heights'
50 'heightList': 'heights'
51 }
51 }
52 }
52 }
53
53
54 desc = {
54 desc = {
55 'Data': {
55 'Data': {
56 'data_output': 'winds',
56 'data_output': 'winds',
57 'utctime': 'timestamps'
57 'utctime': 'timestamps'
58 },
58 },
59 'Metadata': {
59 'Metadata': {
60 'heightList': 'heights'
60 'heightList': 'heights'
61 }
61 }
62 }
62 }
63
63
64 extras = {
64 extras = {
65 'timeZone': 300
65 'timeZone': 300
66 }
66 }
67
67
68 reader = project.addReadUnit(
68 reader = project.addReadUnit(
69 name='HDFReader',
69 name='HDFReader',
70 path='/path/to/files',
70 path='/path/to/files',
71 startDate='2019/01/01',
71 startDate='2019/01/01',
72 endDate='2019/01/31',
72 endDate='2019/01/31',
73 startTime='00:00:00',
73 startTime='00:00:00',
74 endTime='23:59:59',
74 endTime='23:59:59',
75 # description=json.dumps(desc),
75 # description=json.dumps(desc),
76 # extras=json.dumps(extras),
76 # extras=json.dumps(extras),
77 )
77 )
78
78
79 ATTENTION:
80 Add attribute:
81
82 utcoffset='-18000'
83 in the last part of reader in order to work in Local Time
84
79 """
85 """
80
86
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
87 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82
88
83 def __init__(self):
89 def __init__(self):
84 ProcessingUnit.__init__(self)
90 ProcessingUnit.__init__(self)
85 self.dataOut = Parameters()
91 self.dataOut = Parameters()
86 self.ext = ".hdf5"
92 self.ext = ".hdf5"
87 self.optchar = "D"
93 self.optchar = "D"
88 self.meta = {}
94 self.meta = {}
89 self.data = {}
95 self.data = {}
90 self.open_file = h5py.File
96 self.open_file = h5py.File
91 self.open_mode = 'r'
97 self.open_mode = 'r'
92 self.description = {}
98 self.description = {}
93 self.extras = {}
99 self.extras = {}
94 self.filefmt = "*%Y%j***"
100 self.filefmt = "*%Y%j***"
95 self.folderfmt = "*%Y%j"
101 self.folderfmt = "*%Y%j"
96 self.utcoffset = 0
102 self.utcoffset = 0
97
103
98 def setup(self, **kwargs):
104 def setup(self, **kwargs):
99
100 self.set_kwargs(**kwargs)
105 self.set_kwargs(**kwargs)
101 if not self.ext.startswith('.'):
106 if not self.ext.startswith('.'):
102 self.ext = '.{}'.format(self.ext)
107 self.ext = '.{}'.format(self.ext)
103
108
104 if self.online:
109 if self.online:
105 log.log("Searching files in online mode...", self.name)
110 log.log("Searching files in online mode...", self.name)
106
111
107 for nTries in range(self.nTries):
112 for nTries in range(self.nTries):
108 fullpath = self.searchFilesOnLine(self.path, self.startDate,
113 fullpath = self.searchFilesOnLine(self.path, self.startDate,
109 self.endDate, self.expLabel, self.ext, self.walk,
114 self.endDate, self.expLabel, self.ext, self.walk,
110 self.filefmt, self.folderfmt)
115 self.filefmt, self.folderfmt)
111 try:
116 try:
112 fullpath = next(fullpath)
117 fullpath = next(fullpath)
113 except:
118 except:
114 fullpath = None
119 fullpath = None
115
120
116 if fullpath:
121 if fullpath:
117 break
122 break
118
123
119 log.warning(
124 log.warning(
120 'Waiting {} sec for a valid file in {}: try {} ...'.format(
125 'Waiting {} sec for a valid file in {}: try {} ...'.format(
121 self.delay, self.path, nTries + 1),
126 self.delay, self.path, nTries + 1),
122 self.name)
127 self.name)
123 time.sleep(self.delay)
128 time.sleep(self.delay)
124
129
125 if not(fullpath):
130 if not(fullpath):
126 raise schainpy.admin.SchainError(
131 raise schainpy.admin.SchainError(
127 'There isn\'t any valid file in {}'.format(self.path))
132 'There isn\'t any valid file in {}'.format(self.path))
128
133
129 pathname, filename = os.path.split(fullpath)
134 pathname, filename = os.path.split(fullpath)
130 self.year = int(filename[1:5])
135 self.year = int(filename[1:5])
131 self.doy = int(filename[5:8])
136 self.doy = int(filename[5:8])
132 self.set = int(filename[8:11]) - 1
137 self.set = int(filename[8:11]) - 1
133 else:
138 else:
134 log.log("Searching files in {}".format(self.path), self.name)
139 log.log("Searching files in {}".format(self.path), self.name)
135 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
140 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
136 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
141 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
137
142
138 self.setNextFile()
143 self.setNextFile()
139
144
140 return
145 return
141
146
142 def readFirstHeader(self):
147 def readFirstHeader(self):
143 '''Read metadata and data'''
148 '''Read metadata and data'''
144
149
145 self.__readMetadata()
150 self.__readMetadata()
146 self.__readData()
151 self.__readData()
147 self.__setBlockList()
152 self.__setBlockList()
148
153
149 if 'type' in self.meta:
154 if 'type' in self.meta:
150 self.dataOut = eval(self.meta['type'])()
155 self.dataOut = eval(self.meta['type'])()
151
156
152 for attr in self.meta:
157 for attr in self.meta:
153 setattr(self.dataOut, attr, self.meta[attr])
158 setattr(self.dataOut, attr, self.meta[attr])
154
159
155 self.blockIndex = 0
160 self.blockIndex = 0
156
161
157 return
162 return
158
163
159 def __setBlockList(self):
164 def __setBlockList(self):
160 '''
165 '''
161 Selects the data within the times defined
166 Selects the data within the times defined
162
167
163 self.fp
168 self.fp
164 self.startTime
169 self.startTime
165 self.endTime
170 self.endTime
166 self.blockList
171 self.blockList
167 self.blocksPerFile
172 self.blocksPerFile
168
173
169 '''
174 '''
170
175
171 startTime = self.startTime
176 startTime = self.startTime
172 endTime = self.endTime
177 endTime = self.endTime
173 thisUtcTime = self.data['utctime'] + self.utcoffset
178 thisUtcTime = self.data['utctime'] + self.utcoffset
174
179
175 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
180 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
176 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
181 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
177
182
178 thisDate = thisDatetime.date()
183 thisDate = thisDatetime.date()
179 thisTime = thisDatetime.time()
184 thisTime = thisDatetime.time()
180 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
185 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
181 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
186 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
182
183 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
187 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
184 if len(ind)==0:
185 raise schainpy.admin.SchainError("[Reading] Date time selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, self.ext, self.path))
186
188
187 self.blockList = ind
189 self.blockList = ind
188 self.blocksPerFile = len(ind)
190 self.blocksPerFile = len(ind)
191
192 if len(ind)==0:
193 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.blockIndex,
194 self.blocksPerFile,
195 thisDatetime))
196 self.setNextFile()
197 self.readFirstHeader()
198
189 return
199 return
190
200
191 def __readMetadata(self):
201 def __readMetadata(self):
192 '''
202 '''
193 Reads Metadata
203 Reads Metadata
194 '''
204 '''
195
205
196 meta = {}
206 meta = {}
197
207
198 if self.description:
208 if self.description:
199 for key, value in self.description['Metadata'].items():
209 for key, value in self.description['Metadata'].items():
200 meta[key] = self.fp[value][()]
210 meta[key] = self.fp[value][()]
201 else:
211 else:
202 grp = self.fp['Metadata']
212 grp = self.fp['Metadata']
203 for name in grp:
213 for name in grp:
204 meta[name] = grp[name][()]
214 meta[name] = grp[name][()]
205
215
206 if self.extras:
216 if self.extras:
207 for key, value in self.extras.items():
217 for key, value in self.extras.items():
208 meta[key] = value
218 meta[key] = value
209 self.meta = meta
219 self.meta = meta
210
220
211 return
221 return
212
222
213 def __readData(self):
223 def __readData(self):
214
224
215 data = {}
225 data = {}
216
226
217 if self.description:
227 if self.description:
218 for key, value in self.description['Data'].items():
228 for key, value in self.description['Data'].items():
219 if isinstance(value, str):
229 if isinstance(value, str):
220 if isinstance(self.fp[value], h5py.Dataset):
230 if isinstance(self.fp[value], h5py.Dataset):
221 data[key] = self.fp[value][()]
231 data[key] = self.fp[value][()]
222 elif isinstance(self.fp[value], h5py.Group):
232 elif isinstance(self.fp[value], h5py.Group):
223 array = []
233 array = []
224 for ch in self.fp[value]:
234 for ch in self.fp[value]:
225 array.append(self.fp[value][ch][()])
235 array.append(self.fp[value][ch][()])
226 data[key] = numpy.array(array)
236 data[key] = numpy.array(array)
227 elif isinstance(value, list):
237 elif isinstance(value, list):
228 array = []
238 array = []
229 for ch in value:
239 for ch in value:
230 array.append(self.fp[ch][()])
240 array.append(self.fp[ch][()])
231 data[key] = numpy.array(array)
241 data[key] = numpy.array(array)
232 else:
242 else:
233 grp = self.fp['Data']
243 grp = self.fp['Data']
234 for name in grp:
244 for name in grp:
235 if isinstance(grp[name], h5py.Dataset):
245 if isinstance(grp[name], h5py.Dataset):
236 array = grp[name][()]
246 array = grp[name][()]
237 elif isinstance(grp[name], h5py.Group):
247 elif isinstance(grp[name], h5py.Group):
238 array = []
248 array = []
239 for ch in grp[name]:
249 for ch in grp[name]:
240 array.append(grp[name][ch][()])
250 array.append(grp[name][ch][()])
241 array = numpy.array(array)
251 array = numpy.array(array)
242 else:
252 else:
243 log.warning('Unknown type: {}'.format(name))
253 log.warning('Unknown type: {}'.format(name))
244
254
245 if name in self.description:
255 if name in self.description:
246 key = self.description[name]
256 key = self.description[name]
247 else:
257 else:
248 key = name
258 key = name
249 data[key] = array
259 data[key] = array
250
260
251 self.data = data
261 self.data = data
252 return
262 return
253
263
254 def getData(self):
264 def getData(self):
255
265
256 for attr in self.data:
266 for attr in self.data:
257 if self.data[attr].ndim == 1:
267 if self.data[attr].ndim == 1:
258 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
268 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
259 else:
269 else:
260 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
270 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
261
271
262 self.dataOut.flagNoData = False
272 self.dataOut.flagNoData = False
263 self.blockIndex += 1
273 self.blockIndex += 1
264
274
265 log.log("Block No. {}/{} -> {}".format(
275 log.log("Block No. {}/{} -> {}".format(
266 self.blockIndex,
276 self.blockIndex,
267 self.blocksPerFile,
277 self.blocksPerFile,
268 self.dataOut.datatime.ctime()), self.name)
278 self.dataOut.datatime.ctime()), self.name)
269
279
270 return
280 return
271
281
272 def run(self, **kwargs):
282 def run(self, **kwargs):
273
283
274 if not(self.isConfig):
284 if not(self.isConfig):
275 self.setup(**kwargs)
285 self.setup(**kwargs)
276 self.isConfig = True
286 self.isConfig = True
277
287
278 if self.blockIndex == self.blocksPerFile:
288 if self.blockIndex == self.blocksPerFile:
279 self.setNextFile()
289 self.setNextFile()
280
290
281 self.getData()
291 self.getData()
282
292
283 return
293 return
284
294
285 @MPDecorator
295 @MPDecorator
286 class HDFWriter(Operation):
296 class HDFWriter(Operation):
287 """Operation to write HDF5 files.
297 """Operation to write HDF5 files.
288
298
289 The HDF5 file contains by default two groups Data and Metadata where
299 The HDF5 file contains by default two groups Data and Metadata where
290 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
300 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
291 parameters, data attributes are normaly time dependent where the metadata
301 parameters, data attributes are normaly time dependent where the metadata
292 are not.
302 are not.
293 It is possible to customize the structure of the HDF5 file with the
303 It is possible to customize the structure of the HDF5 file with the
294 optional description parameter see the examples.
304 optional description parameter see the examples.
295
305
296 Parameters:
306 Parameters:
297 -----------
307 -----------
298 path : str
308 path : str
299 Path where files will be saved.
309 Path where files will be saved.
300 blocksPerFile : int
310 blocksPerFile : int
301 Number of blocks per file
311 Number of blocks per file
302 metadataList : list
312 metadataList : list
303 List of the dataOut attributes that will be saved as metadata
313 List of the dataOut attributes that will be saved as metadata
304 dataList : int
314 dataList : int
305 List of the dataOut attributes that will be saved as data
315 List of the dataOut attributes that will be saved as data
306 setType : bool
316 setType : bool
307 If True the name of the files corresponds to the timestamp of the data
317 If True the name of the files corresponds to the timestamp of the data
308 description : dict, optional
318 description : dict, optional
309 Dictionary with the desired description of the HDF5 file
319 Dictionary with the desired description of the HDF5 file
310
320
311 Examples
321 Examples
312 --------
322 --------
313
323
314 desc = {
324 desc = {
315 'data_output': {'winds': ['z', 'w', 'v']},
325 'data_output': {'winds': ['z', 'w', 'v']},
316 'utctime': 'timestamps',
326 'utctime': 'timestamps',
317 'heightList': 'heights'
327 'heightList': 'heights'
318 }
328 }
319 desc = {
329 desc = {
320 'data_output': ['z', 'w', 'v'],
330 'data_output': ['z', 'w', 'v'],
321 'utctime': 'timestamps',
331 'utctime': 'timestamps',
322 'heightList': 'heights'
332 'heightList': 'heights'
323 }
333 }
324 desc = {
334 desc = {
325 'Data': {
335 'Data': {
326 'data_output': 'winds',
336 'data_output': 'winds',
327 'utctime': 'timestamps'
337 'utctime': 'timestamps'
328 },
338 },
329 'Metadata': {
339 'Metadata': {
330 'heightList': 'heights'
340 'heightList': 'heights'
331 }
341 }
332 }
342 }
333
343
334 writer = proc_unit.addOperation(name='HDFWriter')
344 writer = proc_unit.addOperation(name='HDFWriter')
335 writer.addParameter(name='path', value='/path/to/file')
345 writer.addParameter(name='path', value='/path/to/file')
336 writer.addParameter(name='blocksPerFile', value='32')
346 writer.addParameter(name='blocksPerFile', value='32')
337 writer.addParameter(name='metadataList', value='heightList,timeZone')
347 writer.addParameter(name='metadataList', value='heightList,timeZone')
338 writer.addParameter(name='dataList',value='data_output,utctime')
348 writer.addParameter(name='dataList',value='data_output,utctime')
339 # writer.addParameter(name='description',value=json.dumps(desc))
349 # writer.addParameter(name='description',value=json.dumps(desc))
340
350
341 """
351 """
342
352
343 ext = ".hdf5"
353 ext = ".hdf5"
344 optchar = "D"
354 optchar = "D"
345 filename = None
355 filename = None
346 path = None
356 path = None
347 setFile = None
357 setFile = None
348 fp = None
358 fp = None
349 firsttime = True
359 firsttime = True
350 #Configurations
360 #Configurations
351 blocksPerFile = None
361 blocksPerFile = None
352 blockIndex = None
362 blockIndex = None
353 dataOut = None
363 dataOut = None
354 #Data Arrays
364 #Data Arrays
355 dataList = None
365 dataList = None
356 metadataList = None
366 metadataList = None
357 currentDay = None
367 currentDay = None
358 lastTime = None
368 lastTime = None
359
369
360 def __init__(self):
370 def __init__(self):
361
371
362 Operation.__init__(self)
372 Operation.__init__(self)
363 return
373 return
364
374
365 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
375 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
366 self.path = path
376 self.path = path
367 self.blocksPerFile = blocksPerFile
377 self.blocksPerFile = blocksPerFile
368 self.metadataList = metadataList
378 self.metadataList = metadataList
369 self.dataList = [s.strip() for s in dataList]
379 self.dataList = [s.strip() for s in dataList]
370 self.setType = setType
380 self.setType = setType
371 self.description = description
381 self.description = description
372
382
373 if self.metadataList is None:
383 if self.metadataList is None:
374 self.metadataList = self.dataOut.metadata_list
384 self.metadataList = self.dataOut.metadata_list
375
385
376 tableList = []
386 tableList = []
377 dsList = []
387 dsList = []
378
388
379 for i in range(len(self.dataList)):
389 for i in range(len(self.dataList)):
380 dsDict = {}
390 dsDict = {}
381 if hasattr(self.dataOut, self.dataList[i]):
391 if hasattr(self.dataOut, self.dataList[i]):
382 dataAux = getattr(self.dataOut, self.dataList[i])
392 dataAux = getattr(self.dataOut, self.dataList[i])
383 dsDict['variable'] = self.dataList[i]
393 dsDict['variable'] = self.dataList[i]
384 else:
394 else:
385 log.warning('Attribute {} not found in dataOut', self.name)
395 log.warning('Attribute {} not found in dataOut', self.name)
386 continue
396 continue
387
397
388 if dataAux is None:
398 if dataAux is None:
389 continue
399 continue
390 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
400 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
391 dsDict['nDim'] = 0
401 dsDict['nDim'] = 0
392 else:
402 else:
393 dsDict['nDim'] = len(dataAux.shape)
403 dsDict['nDim'] = len(dataAux.shape)
394 dsDict['shape'] = dataAux.shape
404 dsDict['shape'] = dataAux.shape
395 dsDict['dsNumber'] = dataAux.shape[0]
405 dsDict['dsNumber'] = dataAux.shape[0]
396 dsDict['dtype'] = dataAux.dtype
406 dsDict['dtype'] = dataAux.dtype
397
407
398 dsList.append(dsDict)
408 dsList.append(dsDict)
399
409
400 self.dsList = dsList
410 self.dsList = dsList
401 self.currentDay = self.dataOut.datatime.date()
411 self.currentDay = self.dataOut.datatime.date()
402
412
403 def timeFlag(self):
413 def timeFlag(self):
404 currentTime = self.dataOut.utctime
414 currentTime = self.dataOut.utctime
405 timeTuple = time.localtime(currentTime)
415 timeTuple = time.localtime(currentTime)
406 dataDay = timeTuple.tm_yday
416 dataDay = timeTuple.tm_yday
407
417
408 if self.lastTime is None:
418 if self.lastTime is None:
409 self.lastTime = currentTime
419 self.lastTime = currentTime
410 self.currentDay = dataDay
420 self.currentDay = dataDay
411 return False
421 return False
412
422
413 timeDiff = currentTime - self.lastTime
423 timeDiff = currentTime - self.lastTime
414
424
415 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
425 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
416 if dataDay != self.currentDay:
426 if dataDay != self.currentDay:
417 self.currentDay = dataDay
427 self.currentDay = dataDay
418 return True
428 return True
419 elif timeDiff > 3*60*60:
429 elif timeDiff > 3*60*60:
420 self.lastTime = currentTime
430 self.lastTime = currentTime
421 return True
431 return True
422 else:
432 else:
423 self.lastTime = currentTime
433 self.lastTime = currentTime
424 return False
434 return False
425
435
426 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
436 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
427 dataList=[], setType=None, description={}):
437 dataList=[], setType=None, description={}):
428
438
429 self.dataOut = dataOut
439 self.dataOut = dataOut
430 if not(self.isConfig):
440 if not(self.isConfig):
431 self.setup(path=path, blocksPerFile=blocksPerFile,
441 self.setup(path=path, blocksPerFile=blocksPerFile,
432 metadataList=metadataList, dataList=dataList,
442 metadataList=metadataList, dataList=dataList,
433 setType=setType, description=description)
443 setType=setType, description=description)
434
444
435 self.isConfig = True
445 self.isConfig = True
436 self.setNextFile()
446 self.setNextFile()
437
447
438 self.putData()
448 self.putData()
439 return
449 return
440
450
441 def setNextFile(self):
451 def setNextFile(self):
442
452
443 ext = self.ext
453 ext = self.ext
444 path = self.path
454 path = self.path
445 setFile = self.setFile
455 setFile = self.setFile
446
456
447 timeTuple = time.localtime(self.dataOut.utctime)
457 timeTuple = time.localtime(self.dataOut.utctime)
448 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
458 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
449 fullpath = os.path.join(path, subfolder)
459 fullpath = os.path.join(path, subfolder)
450
460
451 if os.path.exists(fullpath):
461 if os.path.exists(fullpath):
452 filesList = os.listdir(fullpath)
462 filesList = os.listdir(fullpath)
453 filesList = [k for k in filesList if k.startswith(self.optchar)]
463 filesList = [k for k in filesList if k.startswith(self.optchar)]
454 if len( filesList ) > 0:
464 if len( filesList ) > 0:
455 filesList = sorted(filesList, key=str.lower)
465 filesList = sorted(filesList, key=str.lower)
456 filen = filesList[-1]
466 filen = filesList[-1]
457 # el filename debera tener el siguiente formato
467 # el filename debera tener el siguiente formato
458 # 0 1234 567 89A BCDE (hex)
468 # 0 1234 567 89A BCDE (hex)
459 # x YYYY DDD SSS .ext
469 # x YYYY DDD SSS .ext
460 if isNumber(filen[8:11]):
470 if isNumber(filen[8:11]):
461 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
471 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
462 else:
472 else:
463 setFile = -1
473 setFile = -1
464 else:
474 else:
465 setFile = -1 #inicializo mi contador de seteo
475 setFile = -1 #inicializo mi contador de seteo
466 else:
476 else:
467 os.makedirs(fullpath)
477 os.makedirs(fullpath)
468 setFile = -1 #inicializo mi contador de seteo
478 setFile = -1 #inicializo mi contador de seteo
469
479
470 if self.setType is None:
480 if self.setType is None:
471 setFile += 1
481 setFile += 1
472 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
482 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
473 timeTuple.tm_year,
483 timeTuple.tm_year,
474 timeTuple.tm_yday,
484 timeTuple.tm_yday,
475 setFile,
485 setFile,
476 ext )
486 ext )
477 else:
487 else:
478 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
488 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
479 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
489 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
480 timeTuple.tm_year,
490 timeTuple.tm_year,
481 timeTuple.tm_yday,
491 timeTuple.tm_yday,
482 setFile,
492 setFile,
483 ext )
493 ext )
484
494
485 self.filename = os.path.join( path, subfolder, file )
495 self.filename = os.path.join( path, subfolder, file )
486
496
487 #Setting HDF5 File
497 #Setting HDF5 File
488 self.fp = h5py.File(self.filename, 'w')
498 self.fp = h5py.File(self.filename, 'w')
489 #write metadata
499 #write metadata
490 self.writeMetadata(self.fp)
500 self.writeMetadata(self.fp)
491 #Write data
501 #Write data
492 self.writeData(self.fp)
502 self.writeData(self.fp)
493
503
494 def getLabel(self, name, x=None):
504 def getLabel(self, name, x=None):
495
505
496 if x is None:
506 if x is None:
497 if 'Data' in self.description:
507 if 'Data' in self.description:
498 data = self.description['Data']
508 data = self.description['Data']
499 if 'Metadata' in self.description:
509 if 'Metadata' in self.description:
500 data.update(self.description['Metadata'])
510 data.update(self.description['Metadata'])
501 else:
511 else:
502 data = self.description
512 data = self.description
503 if name in data:
513 if name in data:
504 if isinstance(data[name], str):
514 if isinstance(data[name], str):
505 return data[name]
515 return data[name]
506 elif isinstance(data[name], list):
516 elif isinstance(data[name], list):
507 return None
517 return None
508 elif isinstance(data[name], dict):
518 elif isinstance(data[name], dict):
509 for key, value in data[name].items():
519 for key, value in data[name].items():
510 return key
520 return key
511 return name
521 return name
512 else:
522 else:
513 if 'Metadata' in self.description:
523 if 'Metadata' in self.description:
514 meta = self.description['Metadata']
524 meta = self.description['Metadata']
515 else:
525 else:
516 meta = self.description
526 meta = self.description
517 if name in meta:
527 if name in meta:
518 if isinstance(meta[name], list):
528 if isinstance(meta[name], list):
519 return meta[name][x]
529 return meta[name][x]
520 elif isinstance(meta[name], dict):
530 elif isinstance(meta[name], dict):
521 for key, value in meta[name].items():
531 for key, value in meta[name].items():
522 return value[x]
532 return value[x]
523 if 'cspc' in name:
533 if 'cspc' in name:
524 return 'pair{:02d}'.format(x)
534 return 'pair{:02d}'.format(x)
525 else:
535 else:
526 return 'channel{:02d}'.format(x)
536 return 'channel{:02d}'.format(x)
527
537
528 def writeMetadata(self, fp):
538 def writeMetadata(self, fp):
529
539
530 if self.description:
540 if self.description:
531 if 'Metadata' in self.description:
541 if 'Metadata' in self.description:
532 grp = fp.create_group('Metadata')
542 grp = fp.create_group('Metadata')
533 else:
543 else:
534 grp = fp
544 grp = fp
535 else:
545 else:
536 grp = fp.create_group('Metadata')
546 grp = fp.create_group('Metadata')
537
547
538 for i in range(len(self.metadataList)):
548 for i in range(len(self.metadataList)):
539 if not hasattr(self.dataOut, self.metadataList[i]):
549 if not hasattr(self.dataOut, self.metadataList[i]):
540 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
550 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
541 continue
551 continue
542 value = getattr(self.dataOut, self.metadataList[i])
552 value = getattr(self.dataOut, self.metadataList[i])
543 if isinstance(value, bool):
553 if isinstance(value, bool):
544 if value is True:
554 if value is True:
545 value = 1
555 value = 1
546 else:
556 else:
547 value = 0
557 value = 0
548 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
558 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
549 return
559 return
550
560
551 def writeData(self, fp):
561 def writeData(self, fp):
552
562
553 if self.description:
563 if self.description:
554 if 'Data' in self.description:
564 if 'Data' in self.description:
555 grp = fp.create_group('Data')
565 grp = fp.create_group('Data')
556 else:
566 else:
557 grp = fp
567 grp = fp
558 else:
568 else:
559 grp = fp.create_group('Data')
569 grp = fp.create_group('Data')
560
570
561 dtsets = []
571 dtsets = []
562 data = []
572 data = []
563
573
564 for dsInfo in self.dsList:
574 for dsInfo in self.dsList:
565 if dsInfo['nDim'] == 0:
575 if dsInfo['nDim'] == 0:
566 ds = grp.create_dataset(
576 ds = grp.create_dataset(
567 self.getLabel(dsInfo['variable']),
577 self.getLabel(dsInfo['variable']),
568 (self.blocksPerFile, ),
578 (self.blocksPerFile, ),
569 chunks=True,
579 chunks=True,
570 dtype=numpy.float64)
580 dtype=numpy.float64)
571 dtsets.append(ds)
581 dtsets.append(ds)
572 data.append((dsInfo['variable'], -1))
582 data.append((dsInfo['variable'], -1))
573 else:
583 else:
574 label = self.getLabel(dsInfo['variable'])
584 label = self.getLabel(dsInfo['variable'])
575 if label is not None:
585 if label is not None:
576 sgrp = grp.create_group(label)
586 sgrp = grp.create_group(label)
577 else:
587 else:
578 sgrp = grp
588 sgrp = grp
579 for i in range(dsInfo['dsNumber']):
589 for i in range(dsInfo['dsNumber']):
580 ds = sgrp.create_dataset(
590 ds = sgrp.create_dataset(
581 self.getLabel(dsInfo['variable'], i),
591 self.getLabel(dsInfo['variable'], i),
582 (self.blocksPerFile, ) + dsInfo['shape'][1:],
592 (self.blocksPerFile, ) + dsInfo['shape'][1:],
583 chunks=True,
593 chunks=True,
584 dtype=dsInfo['dtype'])
594 dtype=dsInfo['dtype'])
585 dtsets.append(ds)
595 dtsets.append(ds)
586 data.append((dsInfo['variable'], i))
596 data.append((dsInfo['variable'], i))
587 fp.flush()
597 fp.flush()
588
598
589 log.log('Creating file: {}'.format(fp.filename), self.name)
599 log.log('Creating file: {}'.format(fp.filename), self.name)
590
600
591 self.ds = dtsets
601 self.ds = dtsets
592 self.data = data
602 self.data = data
593 self.firsttime = True
603 self.firsttime = True
594 self.blockIndex = 0
604 self.blockIndex = 0
595 return
605 return
596
606
597 def putData(self):
607 def putData(self):
598
608
599 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
609 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
600 self.closeFile()
610 self.closeFile()
601 self.setNextFile()
611 self.setNextFile()
602
612
603 for i, ds in enumerate(self.ds):
613 for i, ds in enumerate(self.ds):
604 attr, ch = self.data[i]
614 attr, ch = self.data[i]
605 if ch == -1:
615 if ch == -1:
606 ds[self.blockIndex] = getattr(self.dataOut, attr)
616 ds[self.blockIndex] = getattr(self.dataOut, attr)
607 else:
617 else:
608 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
618 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
609
619
610 self.fp.flush()
620 self.fp.flush()
611 self.blockIndex += 1
621 self.blockIndex += 1
612 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
622 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
613
623
614 return
624 return
615
625
616 def closeFile(self):
626 def closeFile(self):
617
627
618 if self.blockIndex != self.blocksPerFile:
628 if self.blockIndex != self.blocksPerFile:
619 for ds in self.ds:
629 for ds in self.ds:
620 ds.resize(self.blockIndex, axis=0)
630 ds.resize(self.blockIndex, axis=0)
621
631
622 if self.fp:
632 if self.fp:
623 self.fp.flush()
633 self.fp.flush()
624 self.fp.close()
634 self.fp.close()
625
635
626 def close(self):
636 def close(self):
627
637
628 self.closeFile()
638 self.closeFile()
General Comments 0
You need to be logged in to leave comments. Login now