##// END OF EJS Templates
schain
RSS
Chequeo previo de startTime and endTime
Alexander Valdez -
r1687:ec27c2dc20c2
parent child
Show More
Side by Side Unified
Context file:
r1687:ec27c2dc20c2 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,626 +1,628
1 import os
1 import os
2 import time
2 import time
3 import datetime
3 import datetime
4
4
5 import numpy
5 import numpy
6 import h5py
6 import h5py
7
7
8 import schainpy.admin
8 import schainpy.admin
9 from schainpy.model.data.jrodata import *
9 from schainpy.model.data.jrodata import *
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 from schainpy.utils import log
12 from schainpy.utils import log
13
13
14
14
15 class HDFReader(Reader, ProcessingUnit):
15 class HDFReader(Reader, ProcessingUnit):
16 """Processing unit to read HDF5 format files
16 """Processing unit to read HDF5 format files
17
17
18 This unit reads HDF5 files created with `HDFWriter` operation contains
18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 attributes.
20 attributes.
21 It is possible to read any HDF5 file by given the structure in the `description`
21 It is possible to read any HDF5 file by given the structure in the `description`
22 parameter, also you can add extra values to metadata with the parameter `extras`.
22 parameter, also you can add extra values to metadata with the parameter `extras`.
23
23
24 Parameters:
24 Parameters:
25 -----------
25 -----------
26 path : str
26 path : str
27 Path where files are located.
27 Path where files are located.
28 startDate : date
28 startDate : date
29 Start date of the files
29 Start date of the files
30 endDate : list
30 endDate : list
31 End date of the files
31 End date of the files
32 startTime : time
32 startTime : time
33 Start time of the files
33 Start time of the files
34 endTime : time
34 endTime : time
35 End time of the files
35 End time of the files
36 description : dict, optional
36 description : dict, optional
37 Dictionary with the description of the HDF5 file
37 Dictionary with the description of the HDF5 file
38 extras : dict, optional
38 extras : dict, optional
39 Dictionary with extra metadata to be be added to `dataOut`
39 Dictionary with extra metadata to be be added to `dataOut`
40
40
41 Examples
41 Examples
42 --------
42 --------
43
43
44 desc = {
44 desc = {
45 'Data': {
45 'Data': {
46 'data_output': ['u', 'v', 'w'],
46 'data_output': ['u', 'v', 'w'],
47 'utctime': 'timestamps',
47 'utctime': 'timestamps',
48 } ,
48 } ,
49 'Metadata': {
49 'Metadata': {
50 'heightList': 'heights'
50 'heightList': 'heights'
51 }
51 }
52 }
52 }
53
53
54 desc = {
54 desc = {
55 'Data': {
55 'Data': {
56 'data_output': 'winds',
56 'data_output': 'winds',
57 'utctime': 'timestamps'
57 'utctime': 'timestamps'
58 },
58 },
59 'Metadata': {
59 'Metadata': {
60 'heightList': 'heights'
60 'heightList': 'heights'
61 }
61 }
62 }
62 }
63
63
64 extras = {
64 extras = {
65 'timeZone': 300
65 'timeZone': 300
66 }
66 }
67
67
68 reader = project.addReadUnit(
68 reader = project.addReadUnit(
69 name='HDFReader',
69 name='HDFReader',
70 path='/path/to/files',
70 path='/path/to/files',
71 startDate='2019/01/01',
71 startDate='2019/01/01',
72 endDate='2019/01/31',
72 endDate='2019/01/31',
73 startTime='00:00:00',
73 startTime='00:00:00',
74 endTime='23:59:59',
74 endTime='23:59:59',
75 # description=json.dumps(desc),
75 # description=json.dumps(desc),
76 # extras=json.dumps(extras),
76 # extras=json.dumps(extras),
77 )
77 )
78
78
79 """
79 """
80
80
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82
82
83 def __init__(self):
83 def __init__(self):
84 ProcessingUnit.__init__(self)
84 ProcessingUnit.__init__(self)
85 self.dataOut = Parameters()
85 self.dataOut = Parameters()
86 self.ext = ".hdf5"
86 self.ext = ".hdf5"
87 self.optchar = "D"
87 self.optchar = "D"
88 self.meta = {}
88 self.meta = {}
89 self.data = {}
89 self.data = {}
90 self.open_file = h5py.File
90 self.open_file = h5py.File
91 self.open_mode = 'r'
91 self.open_mode = 'r'
92 self.description = {}
92 self.description = {}
93 self.extras = {}
93 self.extras = {}
94 self.filefmt = "*%Y%j***"
94 self.filefmt = "*%Y%j***"
95 self.folderfmt = "*%Y%j"
95 self.folderfmt = "*%Y%j"
96 self.utcoffset = 0
96 self.utcoffset = 0
97
97
98 def setup(self, **kwargs):
98 def setup(self, **kwargs):
99
99
100 self.set_kwargs(**kwargs)
100 self.set_kwargs(**kwargs)
101 if not self.ext.startswith('.'):
101 if not self.ext.startswith('.'):
102 self.ext = '.{}'.format(self.ext)
102 self.ext = '.{}'.format(self.ext)
103
103
104 if self.online:
104 if self.online:
105 log.log("Searching files in online mode...", self.name)
105 log.log("Searching files in online mode...", self.name)
106
106
107 for nTries in range(self.nTries):
107 for nTries in range(self.nTries):
108 fullpath = self.searchFilesOnLine(self.path, self.startDate,
108 fullpath = self.searchFilesOnLine(self.path, self.startDate,
109 self.endDate, self.expLabel, self.ext, self.walk,
109 self.endDate, self.expLabel, self.ext, self.walk,
110 self.filefmt, self.folderfmt)
110 self.filefmt, self.folderfmt)
111 try:
111 try:
112 fullpath = next(fullpath)
112 fullpath = next(fullpath)
113 except:
113 except:
114 fullpath = None
114 fullpath = None
115
115
116 if fullpath:
116 if fullpath:
117 break
117 break
118
118
119 log.warning(
119 log.warning(
120 'Waiting {} sec for a valid file in {}: try {} ...'.format(
120 'Waiting {} sec for a valid file in {}: try {} ...'.format(
121 self.delay, self.path, nTries + 1),
121 self.delay, self.path, nTries + 1),
122 self.name)
122 self.name)
123 time.sleep(self.delay)
123 time.sleep(self.delay)
124
124
125 if not(fullpath):
125 if not(fullpath):
126 raise schainpy.admin.SchainError(
126 raise schainpy.admin.SchainError(
127 'There isn\'t any valid file in {}'.format(self.path))
127 'There isn\'t any valid file in {}'.format(self.path))
128
128
129 pathname, filename = os.path.split(fullpath)
129 pathname, filename = os.path.split(fullpath)
130 self.year = int(filename[1:5])
130 self.year = int(filename[1:5])
131 self.doy = int(filename[5:8])
131 self.doy = int(filename[5:8])
132 self.set = int(filename[8:11]) - 1
132 self.set = int(filename[8:11]) - 1
133 else:
133 else:
134 log.log("Searching files in {}".format(self.path), self.name)
134 log.log("Searching files in {}".format(self.path), self.name)
135 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
135 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
136 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
136 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
137
137
138 self.setNextFile()
138 self.setNextFile()
139
139
140 return
140 return
141
141
142 def readFirstHeader(self):
142 def readFirstHeader(self):
143 '''Read metadata and data'''
143 '''Read metadata and data'''
144
144
145 self.__readMetadata()
145 self.__readMetadata()
146 self.__readData()
146 self.__readData()
147 self.__setBlockList()
147 self.__setBlockList()
148
148
149 if 'type' in self.meta:
149 if 'type' in self.meta:
150 self.dataOut = eval(self.meta['type'])()
150 self.dataOut = eval(self.meta['type'])()
151
151
152 for attr in self.meta:
152 for attr in self.meta:
153 setattr(self.dataOut, attr, self.meta[attr])
153 setattr(self.dataOut, attr, self.meta[attr])
154
154
155 self.blockIndex = 0
155 self.blockIndex = 0
156
156
157 return
157 return
158
158
159 def __setBlockList(self):
159 def __setBlockList(self):
160 '''
160 '''
161 Selects the data within the times defined
161 Selects the data within the times defined
162
162
163 self.fp
163 self.fp
164 self.startTime
164 self.startTime
165 self.endTime
165 self.endTime
166 self.blockList
166 self.blockList
167 self.blocksPerFile
167 self.blocksPerFile
168
168
169 '''
169 '''
170
170
171 startTime = self.startTime
171 startTime = self.startTime
172 endTime = self.endTime
172 endTime = self.endTime
173 thisUtcTime = self.data['utctime'] + self.utcoffset
173 thisUtcTime = self.data['utctime'] + self.utcoffset
174
174 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
175 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
175 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
176 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
176
177
177 thisDate = thisDatetime.date()
178 thisDate = thisDatetime.date()
178 thisTime = thisDatetime.time()
179 thisTime = thisDatetime.time()
179
180 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
180 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
181 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
181 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
182
182
183 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
183 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
184 if len(ind)==0:
185 raise schainpy.admin.SchainError("[Reading] Date time selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, self.ext, self.path))
184
186
185 self.blockList = ind
187 self.blockList = ind
186 self.blocksPerFile = len(ind)
188 self.blocksPerFile = len(ind)
187 return
189 return
188
190
189 def __readMetadata(self):
191 def __readMetadata(self):
190 '''
192 '''
191 Reads Metadata
193 Reads Metadata
192 '''
194 '''
193
195
194 meta = {}
196 meta = {}
195
197
196 if self.description:
198 if self.description:
197 for key, value in self.description['Metadata'].items():
199 for key, value in self.description['Metadata'].items():
198 meta[key] = self.fp[value][()]
200 meta[key] = self.fp[value][()]
199 else:
201 else:
200 grp = self.fp['Metadata']
202 grp = self.fp['Metadata']
201 for name in grp:
203 for name in grp:
202 meta[name] = grp[name][()]
204 meta[name] = grp[name][()]
203
205
204 if self.extras:
206 if self.extras:
205 for key, value in self.extras.items():
207 for key, value in self.extras.items():
206 meta[key] = value
208 meta[key] = value
207 self.meta = meta
209 self.meta = meta
208
210
209 return
211 return
210
212
211 def __readData(self):
213 def __readData(self):
212
214
213 data = {}
215 data = {}
214
216
215 if self.description:
217 if self.description:
216 for key, value in self.description['Data'].items():
218 for key, value in self.description['Data'].items():
217 if isinstance(value, str):
219 if isinstance(value, str):
218 if isinstance(self.fp[value], h5py.Dataset):
220 if isinstance(self.fp[value], h5py.Dataset):
219 data[key] = self.fp[value][()]
221 data[key] = self.fp[value][()]
220 elif isinstance(self.fp[value], h5py.Group):
222 elif isinstance(self.fp[value], h5py.Group):
221 array = []
223 array = []
222 for ch in self.fp[value]:
224 for ch in self.fp[value]:
223 array.append(self.fp[value][ch][()])
225 array.append(self.fp[value][ch][()])
224 data[key] = numpy.array(array)
226 data[key] = numpy.array(array)
225 elif isinstance(value, list):
227 elif isinstance(value, list):
226 array = []
228 array = []
227 for ch in value:
229 for ch in value:
228 array.append(self.fp[ch][()])
230 array.append(self.fp[ch][()])
229 data[key] = numpy.array(array)
231 data[key] = numpy.array(array)
230 else:
232 else:
231 grp = self.fp['Data']
233 grp = self.fp['Data']
232 for name in grp:
234 for name in grp:
233 if isinstance(grp[name], h5py.Dataset):
235 if isinstance(grp[name], h5py.Dataset):
234 array = grp[name][()]
236 array = grp[name][()]
235 elif isinstance(grp[name], h5py.Group):
237 elif isinstance(grp[name], h5py.Group):
236 array = []
238 array = []
237 for ch in grp[name]:
239 for ch in grp[name]:
238 array.append(grp[name][ch][()])
240 array.append(grp[name][ch][()])
239 array = numpy.array(array)
241 array = numpy.array(array)
240 else:
242 else:
241 log.warning('Unknown type: {}'.format(name))
243 log.warning('Unknown type: {}'.format(name))
242
244
243 if name in self.description:
245 if name in self.description:
244 key = self.description[name]
246 key = self.description[name]
245 else:
247 else:
246 key = name
248 key = name
247 data[key] = array
249 data[key] = array
248
250
249 self.data = data
251 self.data = data
250 return
252 return
251
253
252 def getData(self):
254 def getData(self):
253
255
254 for attr in self.data:
256 for attr in self.data:
255 if self.data[attr].ndim == 1:
257 if self.data[attr].ndim == 1:
256 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
258 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
257 else:
259 else:
258 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
260 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
259
261
260 self.dataOut.flagNoData = False
262 self.dataOut.flagNoData = False
261 self.blockIndex += 1
263 self.blockIndex += 1
262
264
263 log.log("Block No. {}/{} -> {}".format(
265 log.log("Block No. {}/{} -> {}".format(
264 self.blockIndex,
266 self.blockIndex,
265 self.blocksPerFile,
267 self.blocksPerFile,
266 self.dataOut.datatime.ctime()), self.name)
268 self.dataOut.datatime.ctime()), self.name)
267
269
268 return
270 return
269
271
270 def run(self, **kwargs):
272 def run(self, **kwargs):
271
273
272 if not(self.isConfig):
274 if not(self.isConfig):
273 self.setup(**kwargs)
275 self.setup(**kwargs)
274 self.isConfig = True
276 self.isConfig = True
275
277
276 if self.blockIndex == self.blocksPerFile:
278 if self.blockIndex == self.blocksPerFile:
277 self.setNextFile()
279 self.setNextFile()
278
280
279 self.getData()
281 self.getData()
280
282
281 return
283 return
282
284
283 @MPDecorator
285 @MPDecorator
284 class HDFWriter(Operation):
286 class HDFWriter(Operation):
285 """Operation to write HDF5 files.
287 """Operation to write HDF5 files.
286
288
287 The HDF5 file contains by default two groups Data and Metadata where
289 The HDF5 file contains by default two groups Data and Metadata where
288 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
290 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
289 parameters, data attributes are normaly time dependent where the metadata
291 parameters, data attributes are normaly time dependent where the metadata
290 are not.
292 are not.
291 It is possible to customize the structure of the HDF5 file with the
293 It is possible to customize the structure of the HDF5 file with the
292 optional description parameter see the examples.
294 optional description parameter see the examples.
293
295
294 Parameters:
296 Parameters:
295 -----------
297 -----------
296 path : str
298 path : str
297 Path where files will be saved.
299 Path where files will be saved.
298 blocksPerFile : int
300 blocksPerFile : int
299 Number of blocks per file
301 Number of blocks per file
300 metadataList : list
302 metadataList : list
301 List of the dataOut attributes that will be saved as metadata
303 List of the dataOut attributes that will be saved as metadata
302 dataList : int
304 dataList : int
303 List of the dataOut attributes that will be saved as data
305 List of the dataOut attributes that will be saved as data
304 setType : bool
306 setType : bool
305 If True the name of the files corresponds to the timestamp of the data
307 If True the name of the files corresponds to the timestamp of the data
306 description : dict, optional
308 description : dict, optional
307 Dictionary with the desired description of the HDF5 file
309 Dictionary with the desired description of the HDF5 file
308
310
309 Examples
311 Examples
310 --------
312 --------
311
313
312 desc = {
314 desc = {
313 'data_output': {'winds': ['z', 'w', 'v']},
315 'data_output': {'winds': ['z', 'w', 'v']},
314 'utctime': 'timestamps',
316 'utctime': 'timestamps',
315 'heightList': 'heights'
317 'heightList': 'heights'
316 }
318 }
317 desc = {
319 desc = {
318 'data_output': ['z', 'w', 'v'],
320 'data_output': ['z', 'w', 'v'],
319 'utctime': 'timestamps',
321 'utctime': 'timestamps',
320 'heightList': 'heights'
322 'heightList': 'heights'
321 }
323 }
322 desc = {
324 desc = {
323 'Data': {
325 'Data': {
324 'data_output': 'winds',
326 'data_output': 'winds',
325 'utctime': 'timestamps'
327 'utctime': 'timestamps'
326 },
328 },
327 'Metadata': {
329 'Metadata': {
328 'heightList': 'heights'
330 'heightList': 'heights'
329 }
331 }
330 }
332 }
331
333
332 writer = proc_unit.addOperation(name='HDFWriter')
334 writer = proc_unit.addOperation(name='HDFWriter')
333 writer.addParameter(name='path', value='/path/to/file')
335 writer.addParameter(name='path', value='/path/to/file')
334 writer.addParameter(name='blocksPerFile', value='32')
336 writer.addParameter(name='blocksPerFile', value='32')
335 writer.addParameter(name='metadataList', value='heightList,timeZone')
337 writer.addParameter(name='metadataList', value='heightList,timeZone')
336 writer.addParameter(name='dataList',value='data_output,utctime')
338 writer.addParameter(name='dataList',value='data_output,utctime')
337 # writer.addParameter(name='description',value=json.dumps(desc))
339 # writer.addParameter(name='description',value=json.dumps(desc))
338
340
339 """
341 """
340
342
341 ext = ".hdf5"
343 ext = ".hdf5"
342 optchar = "D"
344 optchar = "D"
343 filename = None
345 filename = None
344 path = None
346 path = None
345 setFile = None
347 setFile = None
346 fp = None
348 fp = None
347 firsttime = True
349 firsttime = True
348 #Configurations
350 #Configurations
349 blocksPerFile = None
351 blocksPerFile = None
350 blockIndex = None
352 blockIndex = None
351 dataOut = None
353 dataOut = None
352 #Data Arrays
354 #Data Arrays
353 dataList = None
355 dataList = None
354 metadataList = None
356 metadataList = None
355 currentDay = None
357 currentDay = None
356 lastTime = None
358 lastTime = None
357
359
358 def __init__(self):
360 def __init__(self):
359
361
360 Operation.__init__(self)
362 Operation.__init__(self)
361 return
363 return
362
364
363 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
365 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
364 self.path = path
366 self.path = path
365 self.blocksPerFile = blocksPerFile
367 self.blocksPerFile = blocksPerFile
366 self.metadataList = metadataList
368 self.metadataList = metadataList
367 self.dataList = [s.strip() for s in dataList]
369 self.dataList = [s.strip() for s in dataList]
368 self.setType = setType
370 self.setType = setType
369 self.description = description
371 self.description = description
370
372
371 if self.metadataList is None:
373 if self.metadataList is None:
372 self.metadataList = self.dataOut.metadata_list
374 self.metadataList = self.dataOut.metadata_list
373
375
374 tableList = []
376 tableList = []
375 dsList = []
377 dsList = []
376
378
377 for i in range(len(self.dataList)):
379 for i in range(len(self.dataList)):
378 dsDict = {}
380 dsDict = {}
379 if hasattr(self.dataOut, self.dataList[i]):
381 if hasattr(self.dataOut, self.dataList[i]):
380 dataAux = getattr(self.dataOut, self.dataList[i])
382 dataAux = getattr(self.dataOut, self.dataList[i])
381 dsDict['variable'] = self.dataList[i]
383 dsDict['variable'] = self.dataList[i]
382 else:
384 else:
383 log.warning('Attribute {} not found in dataOut', self.name)
385 log.warning('Attribute {} not found in dataOut', self.name)
384 continue
386 continue
385
387
386 if dataAux is None:
388 if dataAux is None:
387 continue
389 continue
388 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
390 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
389 dsDict['nDim'] = 0
391 dsDict['nDim'] = 0
390 else:
392 else:
391 dsDict['nDim'] = len(dataAux.shape)
393 dsDict['nDim'] = len(dataAux.shape)
392 dsDict['shape'] = dataAux.shape
394 dsDict['shape'] = dataAux.shape
393 dsDict['dsNumber'] = dataAux.shape[0]
395 dsDict['dsNumber'] = dataAux.shape[0]
394 dsDict['dtype'] = dataAux.dtype
396 dsDict['dtype'] = dataAux.dtype
395
397
396 dsList.append(dsDict)
398 dsList.append(dsDict)
397
399
398 self.dsList = dsList
400 self.dsList = dsList
399 self.currentDay = self.dataOut.datatime.date()
401 self.currentDay = self.dataOut.datatime.date()
400
402
401 def timeFlag(self):
403 def timeFlag(self):
402 currentTime = self.dataOut.utctime
404 currentTime = self.dataOut.utctime
403 timeTuple = time.localtime(currentTime)
405 timeTuple = time.localtime(currentTime)
404 dataDay = timeTuple.tm_yday
406 dataDay = timeTuple.tm_yday
405
407
406 if self.lastTime is None:
408 if self.lastTime is None:
407 self.lastTime = currentTime
409 self.lastTime = currentTime
408 self.currentDay = dataDay
410 self.currentDay = dataDay
409 return False
411 return False
410
412
411 timeDiff = currentTime - self.lastTime
413 timeDiff = currentTime - self.lastTime
412
414
413 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
415 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
414 if dataDay != self.currentDay:
416 if dataDay != self.currentDay:
415 self.currentDay = dataDay
417 self.currentDay = dataDay
416 return True
418 return True
417 elif timeDiff > 3*60*60:
419 elif timeDiff > 3*60*60:
418 self.lastTime = currentTime
420 self.lastTime = currentTime
419 return True
421 return True
420 else:
422 else:
421 self.lastTime = currentTime
423 self.lastTime = currentTime
422 return False
424 return False
423
425
424 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
426 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
425 dataList=[], setType=None, description={}):
427 dataList=[], setType=None, description={}):
426
428
427 self.dataOut = dataOut
429 self.dataOut = dataOut
428 if not(self.isConfig):
430 if not(self.isConfig):
429 self.setup(path=path, blocksPerFile=blocksPerFile,
431 self.setup(path=path, blocksPerFile=blocksPerFile,
430 metadataList=metadataList, dataList=dataList,
432 metadataList=metadataList, dataList=dataList,
431 setType=setType, description=description)
433 setType=setType, description=description)
432
434
433 self.isConfig = True
435 self.isConfig = True
434 self.setNextFile()
436 self.setNextFile()
435
437
436 self.putData()
438 self.putData()
437 return
439 return
438
440
439 def setNextFile(self):
441 def setNextFile(self):
440
442
441 ext = self.ext
443 ext = self.ext
442 path = self.path
444 path = self.path
443 setFile = self.setFile
445 setFile = self.setFile
444
446
445 timeTuple = time.localtime(self.dataOut.utctime)
447 timeTuple = time.localtime(self.dataOut.utctime)
446 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
448 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
447 fullpath = os.path.join(path, subfolder)
449 fullpath = os.path.join(path, subfolder)
448
450
449 if os.path.exists(fullpath):
451 if os.path.exists(fullpath):
450 filesList = os.listdir(fullpath)
452 filesList = os.listdir(fullpath)
451 filesList = [k for k in filesList if k.startswith(self.optchar)]
453 filesList = [k for k in filesList if k.startswith(self.optchar)]
452 if len( filesList ) > 0:
454 if len( filesList ) > 0:
453 filesList = sorted(filesList, key=str.lower)
455 filesList = sorted(filesList, key=str.lower)
454 filen = filesList[-1]
456 filen = filesList[-1]
455 # el filename debera tener el siguiente formato
457 # el filename debera tener el siguiente formato
456 # 0 1234 567 89A BCDE (hex)
458 # 0 1234 567 89A BCDE (hex)
457 # x YYYY DDD SSS .ext
459 # x YYYY DDD SSS .ext
458 if isNumber(filen[8:11]):
460 if isNumber(filen[8:11]):
459 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
461 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
460 else:
462 else:
461 setFile = -1
463 setFile = -1
462 else:
464 else:
463 setFile = -1 #inicializo mi contador de seteo
465 setFile = -1 #inicializo mi contador de seteo
464 else:
466 else:
465 os.makedirs(fullpath)
467 os.makedirs(fullpath)
466 setFile = -1 #inicializo mi contador de seteo
468 setFile = -1 #inicializo mi contador de seteo
467
469
468 if self.setType is None:
470 if self.setType is None:
469 setFile += 1
471 setFile += 1
470 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
472 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
471 timeTuple.tm_year,
473 timeTuple.tm_year,
472 timeTuple.tm_yday,
474 timeTuple.tm_yday,
473 setFile,
475 setFile,
474 ext )
476 ext )
475 else:
477 else:
476 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
478 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
477 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
479 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
478 timeTuple.tm_year,
480 timeTuple.tm_year,
479 timeTuple.tm_yday,
481 timeTuple.tm_yday,
480 setFile,
482 setFile,
481 ext )
483 ext )
482
484
483 self.filename = os.path.join( path, subfolder, file )
485 self.filename = os.path.join( path, subfolder, file )
484
486
485 #Setting HDF5 File
487 #Setting HDF5 File
486 self.fp = h5py.File(self.filename, 'w')
488 self.fp = h5py.File(self.filename, 'w')
487 #write metadata
489 #write metadata
488 self.writeMetadata(self.fp)
490 self.writeMetadata(self.fp)
489 #Write data
491 #Write data
490 self.writeData(self.fp)
492 self.writeData(self.fp)
491
493
492 def getLabel(self, name, x=None):
494 def getLabel(self, name, x=None):
493
495
494 if x is None:
496 if x is None:
495 if 'Data' in self.description:
497 if 'Data' in self.description:
496 data = self.description['Data']
498 data = self.description['Data']
497 if 'Metadata' in self.description:
499 if 'Metadata' in self.description:
498 data.update(self.description['Metadata'])
500 data.update(self.description['Metadata'])
499 else:
501 else:
500 data = self.description
502 data = self.description
501 if name in data:
503 if name in data:
502 if isinstance(data[name], str):
504 if isinstance(data[name], str):
503 return data[name]
505 return data[name]
504 elif isinstance(data[name], list):
506 elif isinstance(data[name], list):
505 return None
507 return None
506 elif isinstance(data[name], dict):
508 elif isinstance(data[name], dict):
507 for key, value in data[name].items():
509 for key, value in data[name].items():
508 return key
510 return key
509 return name
511 return name
510 else:
512 else:
511 if 'Metadata' in self.description:
513 if 'Metadata' in self.description:
512 meta = self.description['Metadata']
514 meta = self.description['Metadata']
513 else:
515 else:
514 meta = self.description
516 meta = self.description
515 if name in meta:
517 if name in meta:
516 if isinstance(meta[name], list):
518 if isinstance(meta[name], list):
517 return meta[name][x]
519 return meta[name][x]
518 elif isinstance(meta[name], dict):
520 elif isinstance(meta[name], dict):
519 for key, value in meta[name].items():
521 for key, value in meta[name].items():
520 return value[x]
522 return value[x]
521 if 'cspc' in name:
523 if 'cspc' in name:
522 return 'pair{:02d}'.format(x)
524 return 'pair{:02d}'.format(x)
523 else:
525 else:
524 return 'channel{:02d}'.format(x)
526 return 'channel{:02d}'.format(x)
525
527
526 def writeMetadata(self, fp):
528 def writeMetadata(self, fp):
527
529
528 if self.description:
530 if self.description:
529 if 'Metadata' in self.description:
531 if 'Metadata' in self.description:
530 grp = fp.create_group('Metadata')
532 grp = fp.create_group('Metadata')
531 else:
533 else:
532 grp = fp
534 grp = fp
533 else:
535 else:
534 grp = fp.create_group('Metadata')
536 grp = fp.create_group('Metadata')
535
537
536 for i in range(len(self.metadataList)):
538 for i in range(len(self.metadataList)):
537 if not hasattr(self.dataOut, self.metadataList[i]):
539 if not hasattr(self.dataOut, self.metadataList[i]):
538 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
540 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
539 continue
541 continue
540 value = getattr(self.dataOut, self.metadataList[i])
542 value = getattr(self.dataOut, self.metadataList[i])
541 if isinstance(value, bool):
543 if isinstance(value, bool):
542 if value is True:
544 if value is True:
543 value = 1
545 value = 1
544 else:
546 else:
545 value = 0
547 value = 0
546 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
548 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
547 return
549 return
548
550
549 def writeData(self, fp):
551 def writeData(self, fp):
550
552
551 if self.description:
553 if self.description:
552 if 'Data' in self.description:
554 if 'Data' in self.description:
553 grp = fp.create_group('Data')
555 grp = fp.create_group('Data')
554 else:
556 else:
555 grp = fp
557 grp = fp
556 else:
558 else:
557 grp = fp.create_group('Data')
559 grp = fp.create_group('Data')
558
560
559 dtsets = []
561 dtsets = []
560 data = []
562 data = []
561
563
562 for dsInfo in self.dsList:
564 for dsInfo in self.dsList:
563 if dsInfo['nDim'] == 0:
565 if dsInfo['nDim'] == 0:
564 ds = grp.create_dataset(
566 ds = grp.create_dataset(
565 self.getLabel(dsInfo['variable']),
567 self.getLabel(dsInfo['variable']),
566 (self.blocksPerFile, ),
568 (self.blocksPerFile, ),
567 chunks=True,
569 chunks=True,
568 dtype=numpy.float64)
570 dtype=numpy.float64)
569 dtsets.append(ds)
571 dtsets.append(ds)
570 data.append((dsInfo['variable'], -1))
572 data.append((dsInfo['variable'], -1))
571 else:
573 else:
572 label = self.getLabel(dsInfo['variable'])
574 label = self.getLabel(dsInfo['variable'])
573 if label is not None:
575 if label is not None:
574 sgrp = grp.create_group(label)
576 sgrp = grp.create_group(label)
575 else:
577 else:
576 sgrp = grp
578 sgrp = grp
577 for i in range(dsInfo['dsNumber']):
579 for i in range(dsInfo['dsNumber']):
578 ds = sgrp.create_dataset(
580 ds = sgrp.create_dataset(
579 self.getLabel(dsInfo['variable'], i),
581 self.getLabel(dsInfo['variable'], i),
580 (self.blocksPerFile, ) + dsInfo['shape'][1:],
582 (self.blocksPerFile, ) + dsInfo['shape'][1:],
581 chunks=True,
583 chunks=True,
582 dtype=dsInfo['dtype'])
584 dtype=dsInfo['dtype'])
583 dtsets.append(ds)
585 dtsets.append(ds)
584 data.append((dsInfo['variable'], i))
586 data.append((dsInfo['variable'], i))
585 fp.flush()
587 fp.flush()
586
588
587 log.log('Creating file: {}'.format(fp.filename), self.name)
589 log.log('Creating file: {}'.format(fp.filename), self.name)
588
590
589 self.ds = dtsets
591 self.ds = dtsets
590 self.data = data
592 self.data = data
591 self.firsttime = True
593 self.firsttime = True
592 self.blockIndex = 0
594 self.blockIndex = 0
593 return
595 return
594
596
595 def putData(self):
597 def putData(self):
596
598
597 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
599 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
598 self.closeFile()
600 self.closeFile()
599 self.setNextFile()
601 self.setNextFile()
600
602
601 for i, ds in enumerate(self.ds):
603 for i, ds in enumerate(self.ds):
602 attr, ch = self.data[i]
604 attr, ch = self.data[i]
603 if ch == -1:
605 if ch == -1:
604 ds[self.blockIndex] = getattr(self.dataOut, attr)
606 ds[self.blockIndex] = getattr(self.dataOut, attr)
605 else:
607 else:
606 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
608 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
607
609
608 self.fp.flush()
610 self.fp.flush()
609 self.blockIndex += 1
611 self.blockIndex += 1
610 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
612 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
611
613
612 return
614 return
613
615
614 def closeFile(self):
616 def closeFile(self):
615
617
616 if self.blockIndex != self.blocksPerFile:
618 if self.blockIndex != self.blocksPerFile:
617 for ds in self.ds:
619 for ds in self.ds:
618 ds.resize(self.blockIndex, axis=0)
620 ds.resize(self.blockIndex, axis=0)
619
621
620 if self.fp:
622 if self.fp:
621 self.fp.flush()
623 self.fp.flush()
622 self.fp.close()
624 self.fp.close()
623
625
624 def close(self):
626 def close(self):
625
627
626 self.closeFile()
628 self.closeFile()
General Comments 0
You need to be logged in to leave comments. Login now