##// END OF EJS Templates
schain
RSS
comentario y recomendacion para usar utcoffset
Alexander Valdez -
r1692:93a8eb01fd3d
parent child
Show More
Side by Side Unified
Context file:
r1692:93a8eb01fd3d Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,638 +1,640
1 import os
1 import os
2 import time
2 import time
3 import datetime
3 import datetime
4
4
5 import numpy
5 import numpy
6 import h5py
6 import h5py
7
7
8 import schainpy.admin
8 import schainpy.admin
9 from schainpy.model.data.jrodata import *
9 from schainpy.model.data.jrodata import *
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 from schainpy.utils import log
12 from schainpy.utils import log
13
13
14
14
15 class HDFReader(Reader, ProcessingUnit):
15 class HDFReader(Reader, ProcessingUnit):
16 """Processing unit to read HDF5 format files
16 """Processing unit to read HDF5 format files
17
17
18 This unit reads HDF5 files created with `HDFWriter` operation contains
18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 attributes.
20 attributes.
21 It is possible to read any HDF5 file by given the structure in the `description`
21 It is possible to read any HDF5 file by given the structure in the `description`
22 parameter, also you can add extra values to metadata with the parameter `extras`.
22 parameter, also you can add extra values to metadata with the parameter `extras`.
23
23
24 Parameters:
24 Parameters:
25 -----------
25 -----------
26 path : str
26 path : str
27 Path where files are located.
27 Path where files are located.
28 startDate : date
28 startDate : date
29 Start date of the files
29 Start date of the files
30 endDate : list
30 endDate : list
31 End date of the files
31 End date of the files
32 startTime : time
32 startTime : time
33 Start time of the files
33 Start time of the files
34 endTime : time
34 endTime : time
35 End time of the files
35 End time of the files
36 description : dict, optional
36 description : dict, optional
37 Dictionary with the description of the HDF5 file
37 Dictionary with the description of the HDF5 file
38 extras : dict, optional
38 extras : dict, optional
39 Dictionary with extra metadata to be be added to `dataOut`
39 Dictionary with extra metadata to be be added to `dataOut`
40
40
41 Attention: Be carefull, add attribute utcoffset, in the last part of reader in order to work in Local Time without time problems.
42
43 -----------
44 utcoffset='-18000'
45
46
41 Examples
47 Examples
42 --------
48 --------
43
49
44 desc = {
50 desc = {
45 'Data': {
51 'Data': {
46 'data_output': ['u', 'v', 'w'],
52 'data_output': ['u', 'v', 'w'],
47 'utctime': 'timestamps',
53 'utctime': 'timestamps',
48 } ,
54 } ,
49 'Metadata': {
55 'Metadata': {
50 'heightList': 'heights'
56 'heightList': 'heights'
51 }
57 }
52 }
58 }
53
59
54 desc = {
60 desc = {
55 'Data': {
61 'Data': {
56 'data_output': 'winds',
62 'data_output': 'winds',
57 'utctime': 'timestamps'
63 'utctime': 'timestamps'
58 },
64 },
59 'Metadata': {
65 'Metadata': {
60 'heightList': 'heights'
66 'heightList': 'heights'
61 }
67 }
62 }
68 }
63
69
64 extras = {
70 extras = {
65 'timeZone': 300
71 'timeZone': 300
66 }
72 }
67
73
74
68 reader = project.addReadUnit(
75 reader = project.addReadUnit(
69 name='HDFReader',
76 name='HDFReader',
70 path='/path/to/files',
77 path='/path/to/files',
71 startDate='2019/01/01',
78 startDate='2019/01/01',
72 endDate='2019/01/31',
79 endDate='2019/01/31',
73 startTime='00:00:00',
80 startTime='00:00:00',
74 endTime='23:59:59',
81 endTime='23:59:59',
82 utcoffset='-18000'
75 # description=json.dumps(desc),
83 # description=json.dumps(desc),
76 # extras=json.dumps(extras),
84 # extras=json.dumps(extras),
77 )
85 )
78
86
79 ATTENTION:
80 Add attribute:
81
82 utcoffset='-18000'
83 in the last part of reader in order to work in Local Time
84
85 """
87 """
86
88
87 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
89 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
88
90
89 def __init__(self):
91 def __init__(self):
90 ProcessingUnit.__init__(self)
92 ProcessingUnit.__init__(self)
91 self.dataOut = Parameters()
93 self.dataOut = Parameters()
92 self.ext = ".hdf5"
94 self.ext = ".hdf5"
93 self.optchar = "D"
95 self.optchar = "D"
94 self.meta = {}
96 self.meta = {}
95 self.data = {}
97 self.data = {}
96 self.open_file = h5py.File
98 self.open_file = h5py.File
97 self.open_mode = 'r'
99 self.open_mode = 'r'
98 self.description = {}
100 self.description = {}
99 self.extras = {}
101 self.extras = {}
100 self.filefmt = "*%Y%j***"
102 self.filefmt = "*%Y%j***"
101 self.folderfmt = "*%Y%j"
103 self.folderfmt = "*%Y%j"
102 self.utcoffset = 0
104 self.utcoffset = 0
103
105
104 def setup(self, **kwargs):
106 def setup(self, **kwargs):
105 self.set_kwargs(**kwargs)
107 self.set_kwargs(**kwargs)
106 if not self.ext.startswith('.'):
108 if not self.ext.startswith('.'):
107 self.ext = '.{}'.format(self.ext)
109 self.ext = '.{}'.format(self.ext)
108
110
109 if self.online:
111 if self.online:
110 log.log("Searching files in online mode...", self.name)
112 log.log("Searching files in online mode...", self.name)
111
113
112 for nTries in range(self.nTries):
114 for nTries in range(self.nTries):
113 fullpath = self.searchFilesOnLine(self.path, self.startDate,
115 fullpath = self.searchFilesOnLine(self.path, self.startDate,
114 self.endDate, self.expLabel, self.ext, self.walk,
116 self.endDate, self.expLabel, self.ext, self.walk,
115 self.filefmt, self.folderfmt)
117 self.filefmt, self.folderfmt)
116 try:
118 try:
117 fullpath = next(fullpath)
119 fullpath = next(fullpath)
118 except:
120 except:
119 fullpath = None
121 fullpath = None
120
122
121 if fullpath:
123 if fullpath:
122 break
124 break
123
125
124 log.warning(
126 log.warning(
125 'Waiting {} sec for a valid file in {}: try {} ...'.format(
127 'Waiting {} sec for a valid file in {}: try {} ...'.format(
126 self.delay, self.path, nTries + 1),
128 self.delay, self.path, nTries + 1),
127 self.name)
129 self.name)
128 time.sleep(self.delay)
130 time.sleep(self.delay)
129
131
130 if not(fullpath):
132 if not(fullpath):
131 raise schainpy.admin.SchainError(
133 raise schainpy.admin.SchainError(
132 'There isn\'t any valid file in {}'.format(self.path))
134 'There isn\'t any valid file in {}'.format(self.path))
133
135
134 pathname, filename = os.path.split(fullpath)
136 pathname, filename = os.path.split(fullpath)
135 self.year = int(filename[1:5])
137 self.year = int(filename[1:5])
136 self.doy = int(filename[5:8])
138 self.doy = int(filename[5:8])
137 self.set = int(filename[8:11]) - 1
139 self.set = int(filename[8:11]) - 1
138 else:
140 else:
139 log.log("Searching files in {}".format(self.path), self.name)
141 log.log("Searching files in {}".format(self.path), self.name)
140 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
142 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
141 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
143 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
142
144
143 self.setNextFile()
145 self.setNextFile()
144
146
145 return
147 return
146
148
147 def readFirstHeader(self):
149 def readFirstHeader(self):
148 '''Read metadata and data'''
150 '''Read metadata and data'''
149
151
150 self.__readMetadata()
152 self.__readMetadata()
151 self.__readData()
153 self.__readData()
152 self.__setBlockList()
154 self.__setBlockList()
153
155
154 if 'type' in self.meta:
156 if 'type' in self.meta:
155 self.dataOut = eval(self.meta['type'])()
157 self.dataOut = eval(self.meta['type'])()
156
158
157 for attr in self.meta:
159 for attr in self.meta:
158 setattr(self.dataOut, attr, self.meta[attr])
160 setattr(self.dataOut, attr, self.meta[attr])
159
161
160 self.blockIndex = 0
162 self.blockIndex = 0
161
163
162 return
164 return
163
165
164 def __setBlockList(self):
166 def __setBlockList(self):
165 '''
167 '''
166 Selects the data within the times defined
168 Selects the data within the times defined
167
169
168 self.fp
170 self.fp
169 self.startTime
171 self.startTime
170 self.endTime
172 self.endTime
171 self.blockList
173 self.blockList
172 self.blocksPerFile
174 self.blocksPerFile
173
175
174 '''
176 '''
175
177
176 startTime = self.startTime
178 startTime = self.startTime
177 endTime = self.endTime
179 endTime = self.endTime
178 thisUtcTime = self.data['utctime'] + self.utcoffset
180 thisUtcTime = self.data['utctime'] + self.utcoffset
179
181
180 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
182 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
181 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
183 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
182
184
183 thisDate = thisDatetime.date()
185 thisDate = thisDatetime.date()
184 thisTime = thisDatetime.time()
186 thisTime = thisDatetime.time()
185 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
187 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
186 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
188 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
187 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
189 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
188
190
189 self.blockList = ind
191 self.blockList = ind
190 self.blocksPerFile = len(ind)
192 self.blocksPerFile = len(ind)
191
193
192 if len(ind)==0:
194 if len(ind)==0:
193 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.blockIndex,
195 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.blockIndex,
194 self.blocksPerFile,
196 self.blocksPerFile,
195 thisDatetime))
197 thisDatetime))
196 self.setNextFile()
198 self.setNextFile()
197 self.readFirstHeader()
199 self.readFirstHeader()
198
200
199 return
201 return
200
202
201 def __readMetadata(self):
203 def __readMetadata(self):
202 '''
204 '''
203 Reads Metadata
205 Reads Metadata
204 '''
206 '''
205
207
206 meta = {}
208 meta = {}
207
209
208 if self.description:
210 if self.description:
209 for key, value in self.description['Metadata'].items():
211 for key, value in self.description['Metadata'].items():
210 meta[key] = self.fp[value][()]
212 meta[key] = self.fp[value][()]
211 else:
213 else:
212 grp = self.fp['Metadata']
214 grp = self.fp['Metadata']
213 for name in grp:
215 for name in grp:
214 meta[name] = grp[name][()]
216 meta[name] = grp[name][()]
215
217
216 if self.extras:
218 if self.extras:
217 for key, value in self.extras.items():
219 for key, value in self.extras.items():
218 meta[key] = value
220 meta[key] = value
219 self.meta = meta
221 self.meta = meta
220
222
221 return
223 return
222
224
223 def __readData(self):
225 def __readData(self):
224
226
225 data = {}
227 data = {}
226
228
227 if self.description:
229 if self.description:
228 for key, value in self.description['Data'].items():
230 for key, value in self.description['Data'].items():
229 if isinstance(value, str):
231 if isinstance(value, str):
230 if isinstance(self.fp[value], h5py.Dataset):
232 if isinstance(self.fp[value], h5py.Dataset):
231 data[key] = self.fp[value][()]
233 data[key] = self.fp[value][()]
232 elif isinstance(self.fp[value], h5py.Group):
234 elif isinstance(self.fp[value], h5py.Group):
233 array = []
235 array = []
234 for ch in self.fp[value]:
236 for ch in self.fp[value]:
235 array.append(self.fp[value][ch][()])
237 array.append(self.fp[value][ch][()])
236 data[key] = numpy.array(array)
238 data[key] = numpy.array(array)
237 elif isinstance(value, list):
239 elif isinstance(value, list):
238 array = []
240 array = []
239 for ch in value:
241 for ch in value:
240 array.append(self.fp[ch][()])
242 array.append(self.fp[ch][()])
241 data[key] = numpy.array(array)
243 data[key] = numpy.array(array)
242 else:
244 else:
243 grp = self.fp['Data']
245 grp = self.fp['Data']
244 for name in grp:
246 for name in grp:
245 if isinstance(grp[name], h5py.Dataset):
247 if isinstance(grp[name], h5py.Dataset):
246 array = grp[name][()]
248 array = grp[name][()]
247 elif isinstance(grp[name], h5py.Group):
249 elif isinstance(grp[name], h5py.Group):
248 array = []
250 array = []
249 for ch in grp[name]:
251 for ch in grp[name]:
250 array.append(grp[name][ch][()])
252 array.append(grp[name][ch][()])
251 array = numpy.array(array)
253 array = numpy.array(array)
252 else:
254 else:
253 log.warning('Unknown type: {}'.format(name))
255 log.warning('Unknown type: {}'.format(name))
254
256
255 if name in self.description:
257 if name in self.description:
256 key = self.description[name]
258 key = self.description[name]
257 else:
259 else:
258 key = name
260 key = name
259 data[key] = array
261 data[key] = array
260
262
261 self.data = data
263 self.data = data
262 return
264 return
263
265
264 def getData(self):
266 def getData(self):
265
267
266 for attr in self.data:
268 for attr in self.data:
267 if self.data[attr].ndim == 1:
269 if self.data[attr].ndim == 1:
268 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
270 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
269 else:
271 else:
270 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
272 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
271
273
272 self.dataOut.flagNoData = False
274 self.dataOut.flagNoData = False
273 self.blockIndex += 1
275 self.blockIndex += 1
274
276
275 log.log("Block No. {}/{} -> {}".format(
277 log.log("Block No. {}/{} -> {}".format(
276 self.blockIndex,
278 self.blockIndex,
277 self.blocksPerFile,
279 self.blocksPerFile,
278 self.dataOut.datatime.ctime()), self.name)
280 self.dataOut.datatime.ctime()), self.name)
279
281
280 return
282 return
281
283
282 def run(self, **kwargs):
284 def run(self, **kwargs):
283
285
284 if not(self.isConfig):
286 if not(self.isConfig):
285 self.setup(**kwargs)
287 self.setup(**kwargs)
286 self.isConfig = True
288 self.isConfig = True
287
289
288 if self.blockIndex == self.blocksPerFile:
290 if self.blockIndex == self.blocksPerFile:
289 self.setNextFile()
291 self.setNextFile()
290
292
291 self.getData()
293 self.getData()
292
294
293 return
295 return
294
296
295 @MPDecorator
297 @MPDecorator
296 class HDFWriter(Operation):
298 class HDFWriter(Operation):
297 """Operation to write HDF5 files.
299 """Operation to write HDF5 files.
298
300
299 The HDF5 file contains by default two groups Data and Metadata where
301 The HDF5 file contains by default two groups Data and Metadata where
300 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
302 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
301 parameters, data attributes are normaly time dependent where the metadata
303 parameters, data attributes are normaly time dependent where the metadata
302 are not.
304 are not.
303 It is possible to customize the structure of the HDF5 file with the
305 It is possible to customize the structure of the HDF5 file with the
304 optional description parameter see the examples.
306 optional description parameter see the examples.
305
307
306 Parameters:
308 Parameters:
307 -----------
309 -----------
308 path : str
310 path : str
309 Path where files will be saved.
311 Path where files will be saved.
310 blocksPerFile : int
312 blocksPerFile : int
311 Number of blocks per file
313 Number of blocks per file
312 metadataList : list
314 metadataList : list
313 List of the dataOut attributes that will be saved as metadata
315 List of the dataOut attributes that will be saved as metadata
314 dataList : int
316 dataList : int
315 List of the dataOut attributes that will be saved as data
317 List of the dataOut attributes that will be saved as data
316 setType : bool
318 setType : bool
317 If True the name of the files corresponds to the timestamp of the data
319 If True the name of the files corresponds to the timestamp of the data
318 description : dict, optional
320 description : dict, optional
319 Dictionary with the desired description of the HDF5 file
321 Dictionary with the desired description of the HDF5 file
320
322
321 Examples
323 Examples
322 --------
324 --------
323
325
324 desc = {
326 desc = {
325 'data_output': {'winds': ['z', 'w', 'v']},
327 'data_output': {'winds': ['z', 'w', 'v']},
326 'utctime': 'timestamps',
328 'utctime': 'timestamps',
327 'heightList': 'heights'
329 'heightList': 'heights'
328 }
330 }
329 desc = {
331 desc = {
330 'data_output': ['z', 'w', 'v'],
332 'data_output': ['z', 'w', 'v'],
331 'utctime': 'timestamps',
333 'utctime': 'timestamps',
332 'heightList': 'heights'
334 'heightList': 'heights'
333 }
335 }
334 desc = {
336 desc = {
335 'Data': {
337 'Data': {
336 'data_output': 'winds',
338 'data_output': 'winds',
337 'utctime': 'timestamps'
339 'utctime': 'timestamps'
338 },
340 },
339 'Metadata': {
341 'Metadata': {
340 'heightList': 'heights'
342 'heightList': 'heights'
341 }
343 }
342 }
344 }
343
345
344 writer = proc_unit.addOperation(name='HDFWriter')
346 writer = proc_unit.addOperation(name='HDFWriter')
345 writer.addParameter(name='path', value='/path/to/file')
347 writer.addParameter(name='path', value='/path/to/file')
346 writer.addParameter(name='blocksPerFile', value='32')
348 writer.addParameter(name='blocksPerFile', value='32')
347 writer.addParameter(name='metadataList', value='heightList,timeZone')
349 writer.addParameter(name='metadataList', value='heightList,timeZone')
348 writer.addParameter(name='dataList',value='data_output,utctime')
350 writer.addParameter(name='dataList',value='data_output,utctime')
349 # writer.addParameter(name='description',value=json.dumps(desc))
351 # writer.addParameter(name='description',value=json.dumps(desc))
350
352
351 """
353 """
352
354
353 ext = ".hdf5"
355 ext = ".hdf5"
354 optchar = "D"
356 optchar = "D"
355 filename = None
357 filename = None
356 path = None
358 path = None
357 setFile = None
359 setFile = None
358 fp = None
360 fp = None
359 firsttime = True
361 firsttime = True
360 #Configurations
362 #Configurations
361 blocksPerFile = None
363 blocksPerFile = None
362 blockIndex = None
364 blockIndex = None
363 dataOut = None
365 dataOut = None
364 #Data Arrays
366 #Data Arrays
365 dataList = None
367 dataList = None
366 metadataList = None
368 metadataList = None
367 currentDay = None
369 currentDay = None
368 lastTime = None
370 lastTime = None
369
371
370 def __init__(self):
372 def __init__(self):
371
373
372 Operation.__init__(self)
374 Operation.__init__(self)
373 return
375 return
374
376
375 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
377 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
376 self.path = path
378 self.path = path
377 self.blocksPerFile = blocksPerFile
379 self.blocksPerFile = blocksPerFile
378 self.metadataList = metadataList
380 self.metadataList = metadataList
379 self.dataList = [s.strip() for s in dataList]
381 self.dataList = [s.strip() for s in dataList]
380 self.setType = setType
382 self.setType = setType
381 self.description = description
383 self.description = description
382
384
383 if self.metadataList is None:
385 if self.metadataList is None:
384 self.metadataList = self.dataOut.metadata_list
386 self.metadataList = self.dataOut.metadata_list
385
387
386 tableList = []
388 tableList = []
387 dsList = []
389 dsList = []
388
390
389 for i in range(len(self.dataList)):
391 for i in range(len(self.dataList)):
390 dsDict = {}
392 dsDict = {}
391 if hasattr(self.dataOut, self.dataList[i]):
393 if hasattr(self.dataOut, self.dataList[i]):
392 dataAux = getattr(self.dataOut, self.dataList[i])
394 dataAux = getattr(self.dataOut, self.dataList[i])
393 dsDict['variable'] = self.dataList[i]
395 dsDict['variable'] = self.dataList[i]
394 else:
396 else:
395 log.warning('Attribute {} not found in dataOut', self.name)
397 log.warning('Attribute {} not found in dataOut', self.name)
396 continue
398 continue
397
399
398 if dataAux is None:
400 if dataAux is None:
399 continue
401 continue
400 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
402 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
401 dsDict['nDim'] = 0
403 dsDict['nDim'] = 0
402 else:
404 else:
403 dsDict['nDim'] = len(dataAux.shape)
405 dsDict['nDim'] = len(dataAux.shape)
404 dsDict['shape'] = dataAux.shape
406 dsDict['shape'] = dataAux.shape
405 dsDict['dsNumber'] = dataAux.shape[0]
407 dsDict['dsNumber'] = dataAux.shape[0]
406 dsDict['dtype'] = dataAux.dtype
408 dsDict['dtype'] = dataAux.dtype
407
409
408 dsList.append(dsDict)
410 dsList.append(dsDict)
409
411
410 self.dsList = dsList
412 self.dsList = dsList
411 self.currentDay = self.dataOut.datatime.date()
413 self.currentDay = self.dataOut.datatime.date()
412
414
413 def timeFlag(self):
415 def timeFlag(self):
414 currentTime = self.dataOut.utctime
416 currentTime = self.dataOut.utctime
415 timeTuple = time.localtime(currentTime)
417 timeTuple = time.localtime(currentTime)
416 dataDay = timeTuple.tm_yday
418 dataDay = timeTuple.tm_yday
417
419
418 if self.lastTime is None:
420 if self.lastTime is None:
419 self.lastTime = currentTime
421 self.lastTime = currentTime
420 self.currentDay = dataDay
422 self.currentDay = dataDay
421 return False
423 return False
422
424
423 timeDiff = currentTime - self.lastTime
425 timeDiff = currentTime - self.lastTime
424
426
425 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
427 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
426 if dataDay != self.currentDay:
428 if dataDay != self.currentDay:
427 self.currentDay = dataDay
429 self.currentDay = dataDay
428 return True
430 return True
429 elif timeDiff > 3*60*60:
431 elif timeDiff > 3*60*60:
430 self.lastTime = currentTime
432 self.lastTime = currentTime
431 return True
433 return True
432 else:
434 else:
433 self.lastTime = currentTime
435 self.lastTime = currentTime
434 return False
436 return False
435
437
436 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
438 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
437 dataList=[], setType=None, description={}):
439 dataList=[], setType=None, description={}):
438
440
439 self.dataOut = dataOut
441 self.dataOut = dataOut
440 if not(self.isConfig):
442 if not(self.isConfig):
441 self.setup(path=path, blocksPerFile=blocksPerFile,
443 self.setup(path=path, blocksPerFile=blocksPerFile,
442 metadataList=metadataList, dataList=dataList,
444 metadataList=metadataList, dataList=dataList,
443 setType=setType, description=description)
445 setType=setType, description=description)
444
446
445 self.isConfig = True
447 self.isConfig = True
446 self.setNextFile()
448 self.setNextFile()
447
449
448 self.putData()
450 self.putData()
449 return
451 return
450
452
451 def setNextFile(self):
453 def setNextFile(self):
452
454
453 ext = self.ext
455 ext = self.ext
454 path = self.path
456 path = self.path
455 setFile = self.setFile
457 setFile = self.setFile
456
458
457 timeTuple = time.localtime(self.dataOut.utctime)
459 timeTuple = time.localtime(self.dataOut.utctime)
458 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
460 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
459 fullpath = os.path.join(path, subfolder)
461 fullpath = os.path.join(path, subfolder)
460
462
461 if os.path.exists(fullpath):
463 if os.path.exists(fullpath):
462 filesList = os.listdir(fullpath)
464 filesList = os.listdir(fullpath)
463 filesList = [k for k in filesList if k.startswith(self.optchar)]
465 filesList = [k for k in filesList if k.startswith(self.optchar)]
464 if len( filesList ) > 0:
466 if len( filesList ) > 0:
465 filesList = sorted(filesList, key=str.lower)
467 filesList = sorted(filesList, key=str.lower)
466 filen = filesList[-1]
468 filen = filesList[-1]
467 # el filename debera tener el siguiente formato
469 # el filename debera tener el siguiente formato
468 # 0 1234 567 89A BCDE (hex)
470 # 0 1234 567 89A BCDE (hex)
469 # x YYYY DDD SSS .ext
471 # x YYYY DDD SSS .ext
470 if isNumber(filen[8:11]):
472 if isNumber(filen[8:11]):
471 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
473 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
472 else:
474 else:
473 setFile = -1
475 setFile = -1
474 else:
476 else:
475 setFile = -1 #inicializo mi contador de seteo
477 setFile = -1 #inicializo mi contador de seteo
476 else:
478 else:
477 os.makedirs(fullpath)
479 os.makedirs(fullpath)
478 setFile = -1 #inicializo mi contador de seteo
480 setFile = -1 #inicializo mi contador de seteo
479
481
480 if self.setType is None:
482 if self.setType is None:
481 setFile += 1
483 setFile += 1
482 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
484 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
483 timeTuple.tm_year,
485 timeTuple.tm_year,
484 timeTuple.tm_yday,
486 timeTuple.tm_yday,
485 setFile,
487 setFile,
486 ext )
488 ext )
487 else:
489 else:
488 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
490 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
489 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
491 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
490 timeTuple.tm_year,
492 timeTuple.tm_year,
491 timeTuple.tm_yday,
493 timeTuple.tm_yday,
492 setFile,
494 setFile,
493 ext )
495 ext )
494
496
495 self.filename = os.path.join( path, subfolder, file )
497 self.filename = os.path.join( path, subfolder, file )
496
498
497 #Setting HDF5 File
499 #Setting HDF5 File
498 self.fp = h5py.File(self.filename, 'w')
500 self.fp = h5py.File(self.filename, 'w')
499 #write metadata
501 #write metadata
500 self.writeMetadata(self.fp)
502 self.writeMetadata(self.fp)
501 #Write data
503 #Write data
502 self.writeData(self.fp)
504 self.writeData(self.fp)
503
505
504 def getLabel(self, name, x=None):
506 def getLabel(self, name, x=None):
505
507
506 if x is None:
508 if x is None:
507 if 'Data' in self.description:
509 if 'Data' in self.description:
508 data = self.description['Data']
510 data = self.description['Data']
509 if 'Metadata' in self.description:
511 if 'Metadata' in self.description:
510 data.update(self.description['Metadata'])
512 data.update(self.description['Metadata'])
511 else:
513 else:
512 data = self.description
514 data = self.description
513 if name in data:
515 if name in data:
514 if isinstance(data[name], str):
516 if isinstance(data[name], str):
515 return data[name]
517 return data[name]
516 elif isinstance(data[name], list):
518 elif isinstance(data[name], list):
517 return None
519 return None
518 elif isinstance(data[name], dict):
520 elif isinstance(data[name], dict):
519 for key, value in data[name].items():
521 for key, value in data[name].items():
520 return key
522 return key
521 return name
523 return name
522 else:
524 else:
523 if 'Metadata' in self.description:
525 if 'Metadata' in self.description:
524 meta = self.description['Metadata']
526 meta = self.description['Metadata']
525 else:
527 else:
526 meta = self.description
528 meta = self.description
527 if name in meta:
529 if name in meta:
528 if isinstance(meta[name], list):
530 if isinstance(meta[name], list):
529 return meta[name][x]
531 return meta[name][x]
530 elif isinstance(meta[name], dict):
532 elif isinstance(meta[name], dict):
531 for key, value in meta[name].items():
533 for key, value in meta[name].items():
532 return value[x]
534 return value[x]
533 if 'cspc' in name:
535 if 'cspc' in name:
534 return 'pair{:02d}'.format(x)
536 return 'pair{:02d}'.format(x)
535 else:
537 else:
536 return 'channel{:02d}'.format(x)
538 return 'channel{:02d}'.format(x)
537
539
538 def writeMetadata(self, fp):
540 def writeMetadata(self, fp):
539
541
540 if self.description:
542 if self.description:
541 if 'Metadata' in self.description:
543 if 'Metadata' in self.description:
542 grp = fp.create_group('Metadata')
544 grp = fp.create_group('Metadata')
543 else:
545 else:
544 grp = fp
546 grp = fp
545 else:
547 else:
546 grp = fp.create_group('Metadata')
548 grp = fp.create_group('Metadata')
547
549
548 for i in range(len(self.metadataList)):
550 for i in range(len(self.metadataList)):
549 if not hasattr(self.dataOut, self.metadataList[i]):
551 if not hasattr(self.dataOut, self.metadataList[i]):
550 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
552 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
551 continue
553 continue
552 value = getattr(self.dataOut, self.metadataList[i])
554 value = getattr(self.dataOut, self.metadataList[i])
553 if isinstance(value, bool):
555 if isinstance(value, bool):
554 if value is True:
556 if value is True:
555 value = 1
557 value = 1
556 else:
558 else:
557 value = 0
559 value = 0
558 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
560 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
559 return
561 return
560
562
561 def writeData(self, fp):
563 def writeData(self, fp):
562
564
563 if self.description:
565 if self.description:
564 if 'Data' in self.description:
566 if 'Data' in self.description:
565 grp = fp.create_group('Data')
567 grp = fp.create_group('Data')
566 else:
568 else:
567 grp = fp
569 grp = fp
568 else:
570 else:
569 grp = fp.create_group('Data')
571 grp = fp.create_group('Data')
570
572
571 dtsets = []
573 dtsets = []
572 data = []
574 data = []
573
575
574 for dsInfo in self.dsList:
576 for dsInfo in self.dsList:
575 if dsInfo['nDim'] == 0:
577 if dsInfo['nDim'] == 0:
576 ds = grp.create_dataset(
578 ds = grp.create_dataset(
577 self.getLabel(dsInfo['variable']),
579 self.getLabel(dsInfo['variable']),
578 (self.blocksPerFile, ),
580 (self.blocksPerFile, ),
579 chunks=True,
581 chunks=True,
580 dtype=numpy.float64)
582 dtype=numpy.float64)
581 dtsets.append(ds)
583 dtsets.append(ds)
582 data.append((dsInfo['variable'], -1))
584 data.append((dsInfo['variable'], -1))
583 else:
585 else:
584 label = self.getLabel(dsInfo['variable'])
586 label = self.getLabel(dsInfo['variable'])
585 if label is not None:
587 if label is not None:
586 sgrp = grp.create_group(label)
588 sgrp = grp.create_group(label)
587 else:
589 else:
588 sgrp = grp
590 sgrp = grp
589 for i in range(dsInfo['dsNumber']):
591 for i in range(dsInfo['dsNumber']):
590 ds = sgrp.create_dataset(
592 ds = sgrp.create_dataset(
591 self.getLabel(dsInfo['variable'], i),
593 self.getLabel(dsInfo['variable'], i),
592 (self.blocksPerFile, ) + dsInfo['shape'][1:],
594 (self.blocksPerFile, ) + dsInfo['shape'][1:],
593 chunks=True,
595 chunks=True,
594 dtype=dsInfo['dtype'])
596 dtype=dsInfo['dtype'])
595 dtsets.append(ds)
597 dtsets.append(ds)
596 data.append((dsInfo['variable'], i))
598 data.append((dsInfo['variable'], i))
597 fp.flush()
599 fp.flush()
598
600
599 log.log('Creating file: {}'.format(fp.filename), self.name)
601 log.log('Creating file: {}'.format(fp.filename), self.name)
600
602
601 self.ds = dtsets
603 self.ds = dtsets
602 self.data = data
604 self.data = data
603 self.firsttime = True
605 self.firsttime = True
604 self.blockIndex = 0
606 self.blockIndex = 0
605 return
607 return
606
608
607 def putData(self):
609 def putData(self):
608
610
609 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
611 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
610 self.closeFile()
612 self.closeFile()
611 self.setNextFile()
613 self.setNextFile()
612
614
613 for i, ds in enumerate(self.ds):
615 for i, ds in enumerate(self.ds):
614 attr, ch = self.data[i]
616 attr, ch = self.data[i]
615 if ch == -1:
617 if ch == -1:
616 ds[self.blockIndex] = getattr(self.dataOut, attr)
618 ds[self.blockIndex] = getattr(self.dataOut, attr)
617 else:
619 else:
618 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
620 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
619
621
620 self.fp.flush()
622 self.fp.flush()
621 self.blockIndex += 1
623 self.blockIndex += 1
622 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
624 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
623
625
624 return
626 return
625
627
626 def closeFile(self):
628 def closeFile(self):
627
629
628 if self.blockIndex != self.blocksPerFile:
630 if self.blockIndex != self.blocksPerFile:
629 for ds in self.ds:
631 for ds in self.ds:
630 ds.resize(self.blockIndex, axis=0)
632 ds.resize(self.blockIndex, axis=0)
631
633
632 if self.fp:
634 if self.fp:
633 self.fp.flush()
635 self.fp.flush()
634 self.fp.close()
636 self.fp.close()
635
637
636 def close(self):
638 def close(self):
637
639
638 self.closeFile()
640 self.closeFile()
General Comments 0
You need to be logged in to leave comments. Login now