##// END OF EJS Templates
schain
RSS
update setNextFile internamente llama a la funcion readFirstHeader
Alexander Valdez -
r1695:17b25ecdbe24
parent child
Show More
Side by Side Unified
Context file:
r1695:17b25ecdbe24 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,640 +1,639
1 import os
1 import os
2 import time
2 import time
3 import datetime
3 import datetime
4
4
5 import numpy
5 import numpy
6 import h5py
6 import h5py
7
7
8 import schainpy.admin
8 import schainpy.admin
9 from schainpy.model.data.jrodata import *
9 from schainpy.model.data.jrodata import *
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 from schainpy.utils import log
12 from schainpy.utils import log
13
13
14
14
15 class HDFReader(Reader, ProcessingUnit):
15 class HDFReader(Reader, ProcessingUnit):
16 """Processing unit to read HDF5 format files
16 """Processing unit to read HDF5 format files
17
17
18 This unit reads HDF5 files created with `HDFWriter` operation contains
18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 attributes.
20 attributes.
21 It is possible to read any HDF5 file by given the structure in the `description`
21 It is possible to read any HDF5 file by given the structure in the `description`
22 parameter, also you can add extra values to metadata with the parameter `extras`.
22 parameter, also you can add extra values to metadata with the parameter `extras`.
23
23
24 Parameters:
24 Parameters:
25 -----------
25 -----------
26 path : str
26 path : str
27 Path where files are located.
27 Path where files are located.
28 startDate : date
28 startDate : date
29 Start date of the files
29 Start date of the files
30 endDate : list
30 endDate : list
31 End date of the files
31 End date of the files
32 startTime : time
32 startTime : time
33 Start time of the files
33 Start time of the files
34 endTime : time
34 endTime : time
35 End time of the files
35 End time of the files
36 description : dict, optional
36 description : dict, optional
37 Dictionary with the description of the HDF5 file
37 Dictionary with the description of the HDF5 file
38 extras : dict, optional
38 extras : dict, optional
39 Dictionary with extra metadata to be be added to `dataOut`
39 Dictionary with extra metadata to be be added to `dataOut`
40
40
41 Attention: Be carefull, add attribute utcoffset, in the last part of reader in order to work in Local Time without time problems.
41 Attention: Be carefull, add attribute utcoffset, in the last part of reader in order to work in Local Time without time problems.
42
42
43 -----------
43 -----------
44 utcoffset='-18000'
44 utcoffset='-18000'
45
45
46
46
47 Examples
47 Examples
48 --------
48 --------
49
49
50 desc = {
50 desc = {
51 'Data': {
51 'Data': {
52 'data_output': ['u', 'v', 'w'],
52 'data_output': ['u', 'v', 'w'],
53 'utctime': 'timestamps',
53 'utctime': 'timestamps',
54 } ,
54 } ,
55 'Metadata': {
55 'Metadata': {
56 'heightList': 'heights'
56 'heightList': 'heights'
57 }
57 }
58 }
58 }
59
59
60 desc = {
60 desc = {
61 'Data': {
61 'Data': {
62 'data_output': 'winds',
62 'data_output': 'winds',
63 'utctime': 'timestamps'
63 'utctime': 'timestamps'
64 },
64 },
65 'Metadata': {
65 'Metadata': {
66 'heightList': 'heights'
66 'heightList': 'heights'
67 }
67 }
68 }
68 }
69
69
70 extras = {
70 extras = {
71 'timeZone': 300
71 'timeZone': 300
72 }
72 }
73
73
74
74
75 reader = project.addReadUnit(
75 reader = project.addReadUnit(
76 name='HDFReader',
76 name='HDFReader',
77 path='/path/to/files',
77 path='/path/to/files',
78 startDate='2019/01/01',
78 startDate='2019/01/01',
79 endDate='2019/01/31',
79 endDate='2019/01/31',
80 startTime='00:00:00',
80 startTime='00:00:00',
81 endTime='23:59:59',
81 endTime='23:59:59',
82 utcoffset='-18000'
82 utcoffset='-18000'
83 # description=json.dumps(desc),
83 # description=json.dumps(desc),
84 # extras=json.dumps(extras),
84 # extras=json.dumps(extras),
85 )
85 )
86
86
87 """
87 """
88
88
89 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
89 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
90
90
91 def __init__(self):
91 def __init__(self):
92 ProcessingUnit.__init__(self)
92 ProcessingUnit.__init__(self)
93 self.dataOut = Parameters()
93 self.dataOut = Parameters()
94 self.ext = ".hdf5"
94 self.ext = ".hdf5"
95 self.optchar = "D"
95 self.optchar = "D"
96 self.meta = {}
96 self.meta = {}
97 self.data = {}
97 self.data = {}
98 self.open_file = h5py.File
98 self.open_file = h5py.File
99 self.open_mode = 'r'
99 self.open_mode = 'r'
100 self.description = {}
100 self.description = {}
101 self.extras = {}
101 self.extras = {}
102 self.filefmt = "*%Y%j***"
102 self.filefmt = "*%Y%j***"
103 self.folderfmt = "*%Y%j"
103 self.folderfmt = "*%Y%j"
104 self.utcoffset = 0
104 self.utcoffset = 0
105
105
106 def setup(self, **kwargs):
106 def setup(self, **kwargs):
107 self.set_kwargs(**kwargs)
107 self.set_kwargs(**kwargs)
108 if not self.ext.startswith('.'):
108 if not self.ext.startswith('.'):
109 self.ext = '.{}'.format(self.ext)
109 self.ext = '.{}'.format(self.ext)
110
110
111 if self.online:
111 if self.online:
112 log.log("Searching files in online mode...", self.name)
112 log.log("Searching files in online mode...", self.name)
113
113
114 for nTries in range(self.nTries):
114 for nTries in range(self.nTries):
115 fullpath = self.searchFilesOnLine(self.path, self.startDate,
115 fullpath = self.searchFilesOnLine(self.path, self.startDate,
116 self.endDate, self.expLabel, self.ext, self.walk,
116 self.endDate, self.expLabel, self.ext, self.walk,
117 self.filefmt, self.folderfmt)
117 self.filefmt, self.folderfmt)
118 try:
118 try:
119 fullpath = next(fullpath)
119 fullpath = next(fullpath)
120 except:
120 except:
121 fullpath = None
121 fullpath = None
122
122
123 if fullpath:
123 if fullpath:
124 break
124 break
125
125
126 log.warning(
126 log.warning(
127 'Waiting {} sec for a valid file in {}: try {} ...'.format(
127 'Waiting {} sec for a valid file in {}: try {} ...'.format(
128 self.delay, self.path, nTries + 1),
128 self.delay, self.path, nTries + 1),
129 self.name)
129 self.name)
130 time.sleep(self.delay)
130 time.sleep(self.delay)
131
131
132 if not(fullpath):
132 if not(fullpath):
133 raise schainpy.admin.SchainError(
133 raise schainpy.admin.SchainError(
134 'There isn\'t any valid file in {}'.format(self.path))
134 'There isn\'t any valid file in {}'.format(self.path))
135
135
136 pathname, filename = os.path.split(fullpath)
136 pathname, filename = os.path.split(fullpath)
137 self.year = int(filename[1:5])
137 self.year = int(filename[1:5])
138 self.doy = int(filename[5:8])
138 self.doy = int(filename[5:8])
139 self.set = int(filename[8:11]) - 1
139 self.set = int(filename[8:11]) - 1
140 else:
140 else:
141 log.log("Searching files in {}".format(self.path), self.name)
141 log.log("Searching files in {}".format(self.path), self.name)
142 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
142 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
143 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
143 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
144
144
145 self.setNextFile()
145 self.setNextFile()
146
146
147 return
147 return
148
148
149 def readFirstHeader(self):
149 def readFirstHeader(self):
150 '''Read metadata and data'''
150 '''Read metadata and data'''
151
151
152 self.__readMetadata()
152 self.__readMetadata()
153 self.__readData()
153 self.__readData()
154 self.__setBlockList()
154 self.__setBlockList()
155
155
156 if 'type' in self.meta:
156 if 'type' in self.meta:
157 self.dataOut = eval(self.meta['type'])()
157 self.dataOut = eval(self.meta['type'])()
158
158
159 for attr in self.meta:
159 for attr in self.meta:
160 setattr(self.dataOut, attr, self.meta[attr])
160 setattr(self.dataOut, attr, self.meta[attr])
161
161
162 self.blockIndex = 0
162 self.blockIndex = 0
163
163
164 return
164 return
165
165
166 def __setBlockList(self):
166 def __setBlockList(self):
167 '''
167 '''
168 Selects the data within the times defined
168 Selects the data within the times defined
169
169
170 self.fp
170 self.fp
171 self.startTime
171 self.startTime
172 self.endTime
172 self.endTime
173 self.blockList
173 self.blockList
174 self.blocksPerFile
174 self.blocksPerFile
175
175
176 '''
176 '''
177
177
178 startTime = self.startTime
178 startTime = self.startTime
179 endTime = self.endTime
179 endTime = self.endTime
180 thisUtcTime = self.data['utctime'] + self.utcoffset
180 thisUtcTime = self.data['utctime'] + self.utcoffset
181
181
182 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
182 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
183 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
183 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
184
184
185 thisDate = thisDatetime.date()
185 thisDate = thisDatetime.date()
186 thisTime = thisDatetime.time()
186 thisTime = thisDatetime.time()
187 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
187 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
188 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
188 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
189 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
189 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
190
190
191 self.blockList = ind
191 self.blockList = ind
192 self.blocksPerFile = len(ind)
192 self.blocksPerFile = len(ind)
193
193
194 if len(ind)==0:
194 if len(ind)==0:
195 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.blockIndex,
195 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.blockIndex,
196 self.blocksPerFile,
196 self.blocksPerFile,
197 thisDatetime))
197 thisDatetime))
198 self.setNextFile()
198 self.setNextFile()
199 #self.readFirstHeader()
200
199
201 return
200 return
202
201
203 def __readMetadata(self):
202 def __readMetadata(self):
204 '''
203 '''
205 Reads Metadata
204 Reads Metadata
206 '''
205 '''
207
206
208 meta = {}
207 meta = {}
209
208
210 if self.description:
209 if self.description:
211 for key, value in self.description['Metadata'].items():
210 for key, value in self.description['Metadata'].items():
212 meta[key] = self.fp[value][()]
211 meta[key] = self.fp[value][()]
213 else:
212 else:
214 grp = self.fp['Metadata']
213 grp = self.fp['Metadata']
215 for name in grp:
214 for name in grp:
216 meta[name] = grp[name][()]
215 meta[name] = grp[name][()]
217
216
218 if self.extras:
217 if self.extras:
219 for key, value in self.extras.items():
218 for key, value in self.extras.items():
220 meta[key] = value
219 meta[key] = value
221 self.meta = meta
220 self.meta = meta
222
221
223 return
222 return
224
223
225 def __readData(self):
224 def __readData(self):
226
225
227 data = {}
226 data = {}
228
227
229 if self.description:
228 if self.description:
230 for key, value in self.description['Data'].items():
229 for key, value in self.description['Data'].items():
231 if isinstance(value, str):
230 if isinstance(value, str):
232 if isinstance(self.fp[value], h5py.Dataset):
231 if isinstance(self.fp[value], h5py.Dataset):
233 data[key] = self.fp[value][()]
232 data[key] = self.fp[value][()]
234 elif isinstance(self.fp[value], h5py.Group):
233 elif isinstance(self.fp[value], h5py.Group):
235 array = []
234 array = []
236 for ch in self.fp[value]:
235 for ch in self.fp[value]:
237 array.append(self.fp[value][ch][()])
236 array.append(self.fp[value][ch][()])
238 data[key] = numpy.array(array)
237 data[key] = numpy.array(array)
239 elif isinstance(value, list):
238 elif isinstance(value, list):
240 array = []
239 array = []
241 for ch in value:
240 for ch in value:
242 array.append(self.fp[ch][()])
241 array.append(self.fp[ch][()])
243 data[key] = numpy.array(array)
242 data[key] = numpy.array(array)
244 else:
243 else:
245 grp = self.fp['Data']
244 grp = self.fp['Data']
246 for name in grp:
245 for name in grp:
247 if isinstance(grp[name], h5py.Dataset):
246 if isinstance(grp[name], h5py.Dataset):
248 array = grp[name][()]
247 array = grp[name][()]
249 elif isinstance(grp[name], h5py.Group):
248 elif isinstance(grp[name], h5py.Group):
250 array = []
249 array = []
251 for ch in grp[name]:
250 for ch in grp[name]:
252 array.append(grp[name][ch][()])
251 array.append(grp[name][ch][()])
253 array = numpy.array(array)
252 array = numpy.array(array)
254 else:
253 else:
255 log.warning('Unknown type: {}'.format(name))
254 log.warning('Unknown type: {}'.format(name))
256
255
257 if name in self.description:
256 if name in self.description:
258 key = self.description[name]
257 key = self.description[name]
259 else:
258 else:
260 key = name
259 key = name
261 data[key] = array
260 data[key] = array
262
261
263 self.data = data
262 self.data = data
264 return
263 return
265
264
266 def getData(self):
265 def getData(self):
267
266
268 for attr in self.data:
267 for attr in self.data:
269 if self.data[attr].ndim == 1:
268 if self.data[attr].ndim == 1:
270 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
269 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
271 else:
270 else:
272 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
271 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
273
272
274 self.dataOut.flagNoData = False
273 self.dataOut.flagNoData = False
275 self.blockIndex += 1
274 self.blockIndex += 1
276
275
277 log.log("Block No. {}/{} -> {}".format(
276 log.log("Block No. {}/{} -> {}".format(
278 self.blockIndex,
277 self.blockIndex,
279 self.blocksPerFile,
278 self.blocksPerFile,
280 self.dataOut.datatime.ctime()), self.name)
279 self.dataOut.datatime.ctime()), self.name)
281
280
282 return
281 return
283
282
284 def run(self, **kwargs):
283 def run(self, **kwargs):
285
284
286 if not(self.isConfig):
285 if not(self.isConfig):
287 self.setup(**kwargs)
286 self.setup(**kwargs)
288 self.isConfig = True
287 self.isConfig = True
289
288
290 if self.blockIndex == self.blocksPerFile:
289 if self.blockIndex == self.blocksPerFile:
291 self.setNextFile()
290 self.setNextFile()
292
291
293 self.getData()
292 self.getData()
294
293
295 return
294 return
296
295
297 @MPDecorator
296 @MPDecorator
298 class HDFWriter(Operation):
297 class HDFWriter(Operation):
299 """Operation to write HDF5 files.
298 """Operation to write HDF5 files.
300
299
301 The HDF5 file contains by default two groups Data and Metadata where
300 The HDF5 file contains by default two groups Data and Metadata where
302 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
301 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
303 parameters, data attributes are normaly time dependent where the metadata
302 parameters, data attributes are normaly time dependent where the metadata
304 are not.
303 are not.
305 It is possible to customize the structure of the HDF5 file with the
304 It is possible to customize the structure of the HDF5 file with the
306 optional description parameter see the examples.
305 optional description parameter see the examples.
307
306
308 Parameters:
307 Parameters:
309 -----------
308 -----------
310 path : str
309 path : str
311 Path where files will be saved.
310 Path where files will be saved.
312 blocksPerFile : int
311 blocksPerFile : int
313 Number of blocks per file
312 Number of blocks per file
314 metadataList : list
313 metadataList : list
315 List of the dataOut attributes that will be saved as metadata
314 List of the dataOut attributes that will be saved as metadata
316 dataList : int
315 dataList : int
317 List of the dataOut attributes that will be saved as data
316 List of the dataOut attributes that will be saved as data
318 setType : bool
317 setType : bool
319 If True the name of the files corresponds to the timestamp of the data
318 If True the name of the files corresponds to the timestamp of the data
320 description : dict, optional
319 description : dict, optional
321 Dictionary with the desired description of the HDF5 file
320 Dictionary with the desired description of the HDF5 file
322
321
323 Examples
322 Examples
324 --------
323 --------
325
324
326 desc = {
325 desc = {
327 'data_output': {'winds': ['z', 'w', 'v']},
326 'data_output': {'winds': ['z', 'w', 'v']},
328 'utctime': 'timestamps',
327 'utctime': 'timestamps',
329 'heightList': 'heights'
328 'heightList': 'heights'
330 }
329 }
331 desc = {
330 desc = {
332 'data_output': ['z', 'w', 'v'],
331 'data_output': ['z', 'w', 'v'],
333 'utctime': 'timestamps',
332 'utctime': 'timestamps',
334 'heightList': 'heights'
333 'heightList': 'heights'
335 }
334 }
336 desc = {
335 desc = {
337 'Data': {
336 'Data': {
338 'data_output': 'winds',
337 'data_output': 'winds',
339 'utctime': 'timestamps'
338 'utctime': 'timestamps'
340 },
339 },
341 'Metadata': {
340 'Metadata': {
342 'heightList': 'heights'
341 'heightList': 'heights'
343 }
342 }
344 }
343 }
345
344
346 writer = proc_unit.addOperation(name='HDFWriter')
345 writer = proc_unit.addOperation(name='HDFWriter')
347 writer.addParameter(name='path', value='/path/to/file')
346 writer.addParameter(name='path', value='/path/to/file')
348 writer.addParameter(name='blocksPerFile', value='32')
347 writer.addParameter(name='blocksPerFile', value='32')
349 writer.addParameter(name='metadataList', value='heightList,timeZone')
348 writer.addParameter(name='metadataList', value='heightList,timeZone')
350 writer.addParameter(name='dataList',value='data_output,utctime')
349 writer.addParameter(name='dataList',value='data_output,utctime')
351 # writer.addParameter(name='description',value=json.dumps(desc))
350 # writer.addParameter(name='description',value=json.dumps(desc))
352
351
353 """
352 """
354
353
355 ext = ".hdf5"
354 ext = ".hdf5"
356 optchar = "D"
355 optchar = "D"
357 filename = None
356 filename = None
358 path = None
357 path = None
359 setFile = None
358 setFile = None
360 fp = None
359 fp = None
361 firsttime = True
360 firsttime = True
362 #Configurations
361 #Configurations
363 blocksPerFile = None
362 blocksPerFile = None
364 blockIndex = None
363 blockIndex = None
365 dataOut = None
364 dataOut = None
366 #Data Arrays
365 #Data Arrays
367 dataList = None
366 dataList = None
368 metadataList = None
367 metadataList = None
369 currentDay = None
368 currentDay = None
370 lastTime = None
369 lastTime = None
371
370
372 def __init__(self):
371 def __init__(self):
373
372
374 Operation.__init__(self)
373 Operation.__init__(self)
375 return
374 return
376
375
377 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
376 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
378 self.path = path
377 self.path = path
379 self.blocksPerFile = blocksPerFile
378 self.blocksPerFile = blocksPerFile
380 self.metadataList = metadataList
379 self.metadataList = metadataList
381 self.dataList = [s.strip() for s in dataList]
380 self.dataList = [s.strip() for s in dataList]
382 self.setType = setType
381 self.setType = setType
383 self.description = description
382 self.description = description
384
383
385 if self.metadataList is None:
384 if self.metadataList is None:
386 self.metadataList = self.dataOut.metadata_list
385 self.metadataList = self.dataOut.metadata_list
387
386
388 tableList = []
387 tableList = []
389 dsList = []
388 dsList = []
390
389
391 for i in range(len(self.dataList)):
390 for i in range(len(self.dataList)):
392 dsDict = {}
391 dsDict = {}
393 if hasattr(self.dataOut, self.dataList[i]):
392 if hasattr(self.dataOut, self.dataList[i]):
394 dataAux = getattr(self.dataOut, self.dataList[i])
393 dataAux = getattr(self.dataOut, self.dataList[i])
395 dsDict['variable'] = self.dataList[i]
394 dsDict['variable'] = self.dataList[i]
396 else:
395 else:
397 log.warning('Attribute {} not found in dataOut', self.name)
396 log.warning('Attribute {} not found in dataOut', self.name)
398 continue
397 continue
399
398
400 if dataAux is None:
399 if dataAux is None:
401 continue
400 continue
402 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
401 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
403 dsDict['nDim'] = 0
402 dsDict['nDim'] = 0
404 else:
403 else:
405 dsDict['nDim'] = len(dataAux.shape)
404 dsDict['nDim'] = len(dataAux.shape)
406 dsDict['shape'] = dataAux.shape
405 dsDict['shape'] = dataAux.shape
407 dsDict['dsNumber'] = dataAux.shape[0]
406 dsDict['dsNumber'] = dataAux.shape[0]
408 dsDict['dtype'] = dataAux.dtype
407 dsDict['dtype'] = dataAux.dtype
409
408
410 dsList.append(dsDict)
409 dsList.append(dsDict)
411
410
412 self.dsList = dsList
411 self.dsList = dsList
413 self.currentDay = self.dataOut.datatime.date()
412 self.currentDay = self.dataOut.datatime.date()
414
413
415 def timeFlag(self):
414 def timeFlag(self):
416 currentTime = self.dataOut.utctime
415 currentTime = self.dataOut.utctime
417 timeTuple = time.localtime(currentTime)
416 timeTuple = time.localtime(currentTime)
418 dataDay = timeTuple.tm_yday
417 dataDay = timeTuple.tm_yday
419
418
420 if self.lastTime is None:
419 if self.lastTime is None:
421 self.lastTime = currentTime
420 self.lastTime = currentTime
422 self.currentDay = dataDay
421 self.currentDay = dataDay
423 return False
422 return False
424
423
425 timeDiff = currentTime - self.lastTime
424 timeDiff = currentTime - self.lastTime
426
425
427 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
426 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
428 if dataDay != self.currentDay:
427 if dataDay != self.currentDay:
429 self.currentDay = dataDay
428 self.currentDay = dataDay
430 return True
429 return True
431 elif timeDiff > 3*60*60:
430 elif timeDiff > 3*60*60:
432 self.lastTime = currentTime
431 self.lastTime = currentTime
433 return True
432 return True
434 else:
433 else:
435 self.lastTime = currentTime
434 self.lastTime = currentTime
436 return False
435 return False
437
436
438 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
437 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
439 dataList=[], setType=None, description={}):
438 dataList=[], setType=None, description={}):
440
439
441 self.dataOut = dataOut
440 self.dataOut = dataOut
442 if not(self.isConfig):
441 if not(self.isConfig):
443 self.setup(path=path, blocksPerFile=blocksPerFile,
442 self.setup(path=path, blocksPerFile=blocksPerFile,
444 metadataList=metadataList, dataList=dataList,
443 metadataList=metadataList, dataList=dataList,
445 setType=setType, description=description)
444 setType=setType, description=description)
446
445
447 self.isConfig = True
446 self.isConfig = True
448 self.setNextFile()
447 self.setNextFile()
449
448
450 self.putData()
449 self.putData()
451 return
450 return
452
451
453 def setNextFile(self):
452 def setNextFile(self):
454
453
455 ext = self.ext
454 ext = self.ext
456 path = self.path
455 path = self.path
457 setFile = self.setFile
456 setFile = self.setFile
458
457
459 timeTuple = time.localtime(self.dataOut.utctime)
458 timeTuple = time.localtime(self.dataOut.utctime)
460 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
459 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
461 fullpath = os.path.join(path, subfolder)
460 fullpath = os.path.join(path, subfolder)
462
461
463 if os.path.exists(fullpath):
462 if os.path.exists(fullpath):
464 filesList = os.listdir(fullpath)
463 filesList = os.listdir(fullpath)
465 filesList = [k for k in filesList if k.startswith(self.optchar)]
464 filesList = [k for k in filesList if k.startswith(self.optchar)]
466 if len( filesList ) > 0:
465 if len( filesList ) > 0:
467 filesList = sorted(filesList, key=str.lower)
466 filesList = sorted(filesList, key=str.lower)
468 filen = filesList[-1]
467 filen = filesList[-1]
469 # el filename debera tener el siguiente formato
468 # el filename debera tener el siguiente formato
470 # 0 1234 567 89A BCDE (hex)
469 # 0 1234 567 89A BCDE (hex)
471 # x YYYY DDD SSS .ext
470 # x YYYY DDD SSS .ext
472 if isNumber(filen[8:11]):
471 if isNumber(filen[8:11]):
473 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
472 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
474 else:
473 else:
475 setFile = -1
474 setFile = -1
476 else:
475 else:
477 setFile = -1 #inicializo mi contador de seteo
476 setFile = -1 #inicializo mi contador de seteo
478 else:
477 else:
479 os.makedirs(fullpath)
478 os.makedirs(fullpath)
480 setFile = -1 #inicializo mi contador de seteo
479 setFile = -1 #inicializo mi contador de seteo
481
480
482 if self.setType is None:
481 if self.setType is None:
483 setFile += 1
482 setFile += 1
484 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
483 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
485 timeTuple.tm_year,
484 timeTuple.tm_year,
486 timeTuple.tm_yday,
485 timeTuple.tm_yday,
487 setFile,
486 setFile,
488 ext )
487 ext )
489 else:
488 else:
490 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
489 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
491 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
490 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
492 timeTuple.tm_year,
491 timeTuple.tm_year,
493 timeTuple.tm_yday,
492 timeTuple.tm_yday,
494 setFile,
493 setFile,
495 ext )
494 ext )
496
495
497 self.filename = os.path.join( path, subfolder, file )
496 self.filename = os.path.join( path, subfolder, file )
498
497
499 #Setting HDF5 File
498 #Setting HDF5 File
500 self.fp = h5py.File(self.filename, 'w')
499 self.fp = h5py.File(self.filename, 'w')
501 #write metadata
500 #write metadata
502 self.writeMetadata(self.fp)
501 self.writeMetadata(self.fp)
503 #Write data
502 #Write data
504 self.writeData(self.fp)
503 self.writeData(self.fp)
505
504
506 def getLabel(self, name, x=None):
505 def getLabel(self, name, x=None):
507
506
508 if x is None:
507 if x is None:
509 if 'Data' in self.description:
508 if 'Data' in self.description:
510 data = self.description['Data']
509 data = self.description['Data']
511 if 'Metadata' in self.description:
510 if 'Metadata' in self.description:
512 data.update(self.description['Metadata'])
511 data.update(self.description['Metadata'])
513 else:
512 else:
514 data = self.description
513 data = self.description
515 if name in data:
514 if name in data:
516 if isinstance(data[name], str):
515 if isinstance(data[name], str):
517 return data[name]
516 return data[name]
518 elif isinstance(data[name], list):
517 elif isinstance(data[name], list):
519 return None
518 return None
520 elif isinstance(data[name], dict):
519 elif isinstance(data[name], dict):
521 for key, value in data[name].items():
520 for key, value in data[name].items():
522 return key
521 return key
523 return name
522 return name
524 else:
523 else:
525 if 'Metadata' in self.description:
524 if 'Metadata' in self.description:
526 meta = self.description['Metadata']
525 meta = self.description['Metadata']
527 else:
526 else:
528 meta = self.description
527 meta = self.description
529 if name in meta:
528 if name in meta:
530 if isinstance(meta[name], list):
529 if isinstance(meta[name], list):
531 return meta[name][x]
530 return meta[name][x]
532 elif isinstance(meta[name], dict):
531 elif isinstance(meta[name], dict):
533 for key, value in meta[name].items():
532 for key, value in meta[name].items():
534 return value[x]
533 return value[x]
535 if 'cspc' in name:
534 if 'cspc' in name:
536 return 'pair{:02d}'.format(x)
535 return 'pair{:02d}'.format(x)
537 else:
536 else:
538 return 'channel{:02d}'.format(x)
537 return 'channel{:02d}'.format(x)
539
538
540 def writeMetadata(self, fp):
539 def writeMetadata(self, fp):
541
540
542 if self.description:
541 if self.description:
543 if 'Metadata' in self.description:
542 if 'Metadata' in self.description:
544 grp = fp.create_group('Metadata')
543 grp = fp.create_group('Metadata')
545 else:
544 else:
546 grp = fp
545 grp = fp
547 else:
546 else:
548 grp = fp.create_group('Metadata')
547 grp = fp.create_group('Metadata')
549
548
550 for i in range(len(self.metadataList)):
549 for i in range(len(self.metadataList)):
551 if not hasattr(self.dataOut, self.metadataList[i]):
550 if not hasattr(self.dataOut, self.metadataList[i]):
552 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
551 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
553 continue
552 continue
554 value = getattr(self.dataOut, self.metadataList[i])
553 value = getattr(self.dataOut, self.metadataList[i])
555 if isinstance(value, bool):
554 if isinstance(value, bool):
556 if value is True:
555 if value is True:
557 value = 1
556 value = 1
558 else:
557 else:
559 value = 0
558 value = 0
560 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
559 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
561 return
560 return
562
561
563 def writeData(self, fp):
562 def writeData(self, fp):
564
563
565 if self.description:
564 if self.description:
566 if 'Data' in self.description:
565 if 'Data' in self.description:
567 grp = fp.create_group('Data')
566 grp = fp.create_group('Data')
568 else:
567 else:
569 grp = fp
568 grp = fp
570 else:
569 else:
571 grp = fp.create_group('Data')
570 grp = fp.create_group('Data')
572
571
573 dtsets = []
572 dtsets = []
574 data = []
573 data = []
575
574
576 for dsInfo in self.dsList:
575 for dsInfo in self.dsList:
577 if dsInfo['nDim'] == 0:
576 if dsInfo['nDim'] == 0:
578 ds = grp.create_dataset(
577 ds = grp.create_dataset(
579 self.getLabel(dsInfo['variable']),
578 self.getLabel(dsInfo['variable']),
580 (self.blocksPerFile, ),
579 (self.blocksPerFile, ),
581 chunks=True,
580 chunks=True,
582 dtype=numpy.float64)
581 dtype=numpy.float64)
583 dtsets.append(ds)
582 dtsets.append(ds)
584 data.append((dsInfo['variable'], -1))
583 data.append((dsInfo['variable'], -1))
585 else:
584 else:
586 label = self.getLabel(dsInfo['variable'])
585 label = self.getLabel(dsInfo['variable'])
587 if label is not None:
586 if label is not None:
588 sgrp = grp.create_group(label)
587 sgrp = grp.create_group(label)
589 else:
588 else:
590 sgrp = grp
589 sgrp = grp
591 for i in range(dsInfo['dsNumber']):
590 for i in range(dsInfo['dsNumber']):
592 ds = sgrp.create_dataset(
591 ds = sgrp.create_dataset(
593 self.getLabel(dsInfo['variable'], i),
592 self.getLabel(dsInfo['variable'], i),
594 (self.blocksPerFile, ) + dsInfo['shape'][1:],
593 (self.blocksPerFile, ) + dsInfo['shape'][1:],
595 chunks=True,
594 chunks=True,
596 dtype=dsInfo['dtype'])
595 dtype=dsInfo['dtype'])
597 dtsets.append(ds)
596 dtsets.append(ds)
598 data.append((dsInfo['variable'], i))
597 data.append((dsInfo['variable'], i))
599 fp.flush()
598 fp.flush()
600
599
601 log.log('Creating file: {}'.format(fp.filename), self.name)
600 log.log('Creating file: {}'.format(fp.filename), self.name)
602
601
603 self.ds = dtsets
602 self.ds = dtsets
604 self.data = data
603 self.data = data
605 self.firsttime = True
604 self.firsttime = True
606 self.blockIndex = 0
605 self.blockIndex = 0
607 return
606 return
608
607
609 def putData(self):
608 def putData(self):
610
609
611 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
610 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
612 self.closeFile()
611 self.closeFile()
613 self.setNextFile()
612 self.setNextFile()
614
613
615 for i, ds in enumerate(self.ds):
614 for i, ds in enumerate(self.ds):
616 attr, ch = self.data[i]
615 attr, ch = self.data[i]
617 if ch == -1:
616 if ch == -1:
618 ds[self.blockIndex] = getattr(self.dataOut, attr)
617 ds[self.blockIndex] = getattr(self.dataOut, attr)
619 else:
618 else:
620 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
619 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
621
620
622 self.fp.flush()
621 self.fp.flush()
623 self.blockIndex += 1
622 self.blockIndex += 1
624 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
623 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
625
624
626 return
625 return
627
626
628 def closeFile(self):
627 def closeFile(self):
629
628
630 if self.blockIndex != self.blocksPerFile:
629 if self.blockIndex != self.blocksPerFile:
631 for ds in self.ds:
630 for ds in self.ds:
632 ds.resize(self.blockIndex, axis=0)
631 ds.resize(self.blockIndex, axis=0)
633
632
634 if self.fp:
633 if self.fp:
635 self.fp.flush()
634 self.fp.flush()
636 self.fp.close()
635 self.fp.close()
637
636
638 def close(self):
637 def close(self):
639
638
640 self.closeFile()
639 self.closeFile()
General Comments 0
You need to be logged in to leave comments. Login now