##// END OF EJS Templates
schain
RSS
update uniquechannel
avaldez -
r1784:57448bf73d50
parent child
Show More
Side by Side Unified
Context file:
r1784:57448bf73d50 Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -1,662 +1,662
1 import os
1 import os
2 import time
2 import time
3 import datetime
3 import datetime
4
4
5 import numpy
5 import numpy
6 import h5py
6 import h5py
7
7
8 import schainpy.admin
8 import schainpy.admin
9 from schainpy.model.data.jrodata import *
9 from schainpy.model.data.jrodata import *
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 from schainpy.utils import log
12 from schainpy.utils import log
13
13
14
14
15 class HDFReader(Reader, ProcessingUnit):
15 class HDFReader(Reader, ProcessingUnit):
16 """Processing unit to read HDF5 format files
16 """Processing unit to read HDF5 format files
17
17
18 This unit reads HDF5 files created with `HDFWriter` operation contains
18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 attributes.
20 attributes.
21 It is possible to read any HDF5 file by given the structure in the `description`
21 It is possible to read any HDF5 file by given the structure in the `description`
22 parameter, also you can add extra values to metadata with the parameter `extras`.
22 parameter, also you can add extra values to metadata with the parameter `extras`.
23
23
24 Parameters:
24 Parameters:
25 -----------
25 -----------
26 path : str
26 path : str
27 Path where files are located.
27 Path where files are located.
28 startDate : date
28 startDate : date
29 Start date of the files
29 Start date of the files
30 endDate : list
30 endDate : list
31 End date of the files
31 End date of the files
32 startTime : time
32 startTime : time
33 Start time of the files
33 Start time of the files
34 endTime : time
34 endTime : time
35 End time of the files
35 End time of the files
36 description : dict, optional
36 description : dict, optional
37 Dictionary with the description of the HDF5 file
37 Dictionary with the description of the HDF5 file
38 extras : dict, optional
38 extras : dict, optional
39 Dictionary with extra metadata to be be added to `dataOut`
39 Dictionary with extra metadata to be be added to `dataOut`
40
40
41 Examples
41 Examples
42 --------
42 --------
43
43
44 desc = {
44 desc = {
45 'Data': {
45 'Data': {
46 'data_output': ['u', 'v', 'w'],
46 'data_output': ['u', 'v', 'w'],
47 'utctime': 'timestamps',
47 'utctime': 'timestamps',
48 } ,
48 } ,
49 'Metadata': {
49 'Metadata': {
50 'heightList': 'heights'
50 'heightList': 'heights'
51 }
51 }
52 }
52 }
53
53
54 desc = {
54 desc = {
55 'Data': {
55 'Data': {
56 'data_output': 'winds',
56 'data_output': 'winds',
57 'utctime': 'timestamps'
57 'utctime': 'timestamps'
58 },
58 },
59 'Metadata': {
59 'Metadata': {
60 'heightList': 'heights'
60 'heightList': 'heights'
61 }
61 }
62 }
62 }
63
63
64 extras = {
64 extras = {
65 'timeZone': 300
65 'timeZone': 300
66 }
66 }
67
67
68 reader = project.addReadUnit(
68 reader = project.addReadUnit(
69 name='HDFReader',
69 name='HDFReader',
70 path='/path/to/files',
70 path='/path/to/files',
71 startDate='2019/01/01',
71 startDate='2019/01/01',
72 endDate='2019/01/31',
72 endDate='2019/01/31',
73 startTime='00:00:00',
73 startTime='00:00:00',
74 endTime='23:59:59',
74 endTime='23:59:59',
75 # description=json.dumps(desc),
75 # description=json.dumps(desc),
76 # extras=json.dumps(extras),
76 # extras=json.dumps(extras),
77 )
77 )
78
78
79 """
79 """
80
80
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82
82
83 def __init__(self):
83 def __init__(self):
84 ProcessingUnit.__init__(self)
84 ProcessingUnit.__init__(self)
85 self.dataOut = Parameters()
85 self.dataOut = Parameters()
86 self.ext = ".hdf5"
86 self.ext = ".hdf5"
87 self.optchar = "D"
87 self.optchar = "D"
88 self.meta = {}
88 self.meta = {}
89 self.data = {}
89 self.data = {}
90 self.open_file = h5py.File
90 self.open_file = h5py.File
91 self.open_mode = 'r'
91 self.open_mode = 'r'
92 self.description = {}
92 self.description = {}
93 self.extras = {}
93 self.extras = {}
94 self.filefmt = "*%Y%j***"
94 self.filefmt = "*%Y%j***"
95 self.folderfmt = "*%Y%j"
95 self.folderfmt = "*%Y%j"
96 self.utcoffset = 0
96 self.utcoffset = 0
97
97
98 def setup(self, **kwargs):
98 def setup(self, **kwargs):
99
99
100 self.set_kwargs(**kwargs)
100 self.set_kwargs(**kwargs)
101 if not self.ext.startswith('.'):
101 if not self.ext.startswith('.'):
102 self.ext = '.{}'.format(self.ext)
102 self.ext = '.{}'.format(self.ext)
103
103
104 if self.online:
104 if self.online:
105 log.log("Searching files in online mode...", self.name)
105 log.log("Searching files in online mode...", self.name)
106
106
107 for nTries in range(self.nTries):
107 for nTries in range(self.nTries):
108 fullpath = self.searchFilesOnLine(self.path, self.startDate,
108 fullpath = self.searchFilesOnLine(self.path, self.startDate,
109 self.endDate, self.expLabel, self.ext, self.walk,
109 self.endDate, self.expLabel, self.ext, self.walk,
110 self.filefmt, self.folderfmt)
110 self.filefmt, self.folderfmt)
111 try:
111 try:
112 fullpath = next(fullpath)
112 fullpath = next(fullpath)
113 except:
113 except:
114 fullpath = None
114 fullpath = None
115
115
116 if fullpath:
116 if fullpath:
117 break
117 break
118
118
119 log.warning(
119 log.warning(
120 'Waiting {} sec for a valid file in {}: try {} ...'.format(
120 'Waiting {} sec for a valid file in {}: try {} ...'.format(
121 self.delay, self.path, nTries + 1),
121 self.delay, self.path, nTries + 1),
122 self.name)
122 self.name)
123 time.sleep(self.delay)
123 time.sleep(self.delay)
124
124
125 if not(fullpath):
125 if not(fullpath):
126 raise schainpy.admin.SchainError(
126 raise schainpy.admin.SchainError(
127 'There isn\'t any valid file in {}'.format(self.path))
127 'There isn\'t any valid file in {}'.format(self.path))
128
128
129 pathname, filename = os.path.split(fullpath)
129 pathname, filename = os.path.split(fullpath)
130 self.year = int(filename[1:5])
130 self.year = int(filename[1:5])
131 self.doy = int(filename[5:8])
131 self.doy = int(filename[5:8])
132 self.set = int(filename[8:11]) - 1
132 self.set = int(filename[8:11]) - 1
133 else:
133 else:
134 log.log("Searching files in {}".format(self.path), self.name)
134 log.log("Searching files in {}".format(self.path), self.name)
135 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
135 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
136 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
136 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
137
137
138 self.setNextFile()
138 self.setNextFile()
139
139
140 return
140 return
141
141
142 def readFirstHeader(self):
142 def readFirstHeader(self):
143 '''Read metadata and data'''
143 '''Read metadata and data'''
144
144
145 self.__readMetadata()
145 self.__readMetadata()
146 self.__readData()
146 self.__readData()
147 self.__setBlockList()
147 self.__setBlockList()
148
148
149 if 'type' in self.meta:
149 if 'type' in self.meta:
150 self.dataOut = eval(self.meta['type'])()
150 self.dataOut = eval(self.meta['type'])()
151
151
152 for attr in self.meta:
152 for attr in self.meta:
153 setattr(self.dataOut, attr, self.meta[attr])
153 setattr(self.dataOut, attr, self.meta[attr])
154
154
155 self.blockIndex = 0
155 self.blockIndex = 0
156
156
157 return
157 return
158
158
159 def __setBlockList(self):
159 def __setBlockList(self):
160 '''
160 '''
161 Selects the data within the times defined
161 Selects the data within the times defined
162
162
163 self.fp
163 self.fp
164 self.startTime
164 self.startTime
165 self.endTime
165 self.endTime
166 self.blockList
166 self.blockList
167 self.blocksPerFile
167 self.blocksPerFile
168
168
169 '''
169 '''
170
170
171 startTime = self.startTime
171 startTime = self.startTime
172 endTime = self.endTime
172 endTime = self.endTime
173 thisUtcTime = self.data['utctime'] + self.utcoffset
173 thisUtcTime = self.data['utctime'] + self.utcoffset
174 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
174 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
175 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
175 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
176
176
177 thisDate = thisDatetime.date()
177 thisDate = thisDatetime.date()
178 thisTime = thisDatetime.time()
178 thisTime = thisDatetime.time()
179
179
180 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
180 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
181 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
181 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
182
182
183 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
183 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
184
184
185 self.blockList = ind
185 self.blockList = ind
186 self.blocksPerFile = len(ind)
186 self.blocksPerFile = len(ind)
187 # similar to master
187 # similar to master
188 if len(ind)==0:
188 if len(ind)==0:
189 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.blockIndex,
189 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.blockIndex,
190 self.blocksPerFile,
190 self.blocksPerFile,
191 thisDatetime))
191 thisDatetime))
192 self.setNextFile()
192 self.setNextFile()
193 # similar to master
193 # similar to master
194 return
194 return
195
195
196 def __readMetadata(self):
196 def __readMetadata(self):
197 '''
197 '''
198 Reads Metadata
198 Reads Metadata
199 '''
199 '''
200
200
201 meta = {}
201 meta = {}
202
202
203 if self.description:
203 if self.description:
204 for key, value in self.description['Metadata'].items():
204 for key, value in self.description['Metadata'].items():
205 meta[key] = self.fp[value][()]
205 meta[key] = self.fp[value][()]
206 else:
206 else:
207 grp = self.fp['Metadata']
207 grp = self.fp['Metadata']
208 for name in grp:
208 for name in grp:
209 meta[name] = grp[name][()]
209 meta[name] = grp[name][()]
210
210
211 if self.extras:
211 if self.extras:
212 for key, value in self.extras.items():
212 for key, value in self.extras.items():
213 meta[key] = value
213 meta[key] = value
214 self.meta = meta
214 self.meta = meta
215
215
216 return
216 return
217
217
218 def __readData(self):
218 def __readData(self):
219
219
220 data = {}
220 data = {}
221
221
222 if self.description:
222 if self.description:
223 for key, value in self.description['Data'].items():
223 for key, value in self.description['Data'].items():
224 if isinstance(value, str):
224 if isinstance(value, str):
225 if isinstance(self.fp[value], h5py.Dataset):
225 if isinstance(self.fp[value], h5py.Dataset):
226 data[key] = self.fp[value][()]
226 data[key] = self.fp[value][()]
227 elif isinstance(self.fp[value], h5py.Group):
227 elif isinstance(self.fp[value], h5py.Group):
228 array = []
228 array = []
229 for ch in self.fp[value]:
229 for ch in self.fp[value]:
230 array.append(self.fp[value][ch][()])
230 array.append(self.fp[value][ch][()])
231 data[key] = numpy.array(array)
231 data[key] = numpy.array(array)
232 elif isinstance(value, list):
232 elif isinstance(value, list):
233 array = []
233 array = []
234 for ch in value:
234 for ch in value:
235 array.append(self.fp[ch][()])
235 array.append(self.fp[ch][()])
236 data[key] = numpy.array(array)
236 data[key] = numpy.array(array)
237 else:
237 else:
238 grp = self.fp['Data']
238 grp = self.fp['Data']
239 for name in grp:
239 for name in grp:
240 if isinstance(grp[name], h5py.Dataset):
240 if isinstance(grp[name], h5py.Dataset):
241 array = grp[name][()]
241 array = grp[name][()]
242 elif isinstance(grp[name], h5py.Group):
242 elif isinstance(grp[name], h5py.Group):
243 array = []
243 array = []
244 for ch in grp[name]:
244 for ch in grp[name]:
245 array.append(grp[name][ch][()])
245 array.append(grp[name][ch][()])
246 array = numpy.array(array)
246 array = numpy.array(array)
247 else:
247 else:
248 log.warning('Unknown type: {}'.format(name))
248 log.warning('Unknown type: {}'.format(name))
249
249
250 if name in self.description:
250 if name in self.description:
251 key = self.description[name]
251 key = self.description[name]
252 else:
252 else:
253 key = name
253 key = name
254 data[key] = array
254 data[key] = array
255
255
256 self.data = data
256 self.data = data
257 return
257 return
258
258
259 def getData(self):
259 def getData(self):
260
260
261 for attr in self.data:
261 for attr in self.data:
262 if self.data[attr].ndim == 1:
262 if self.data[attr].ndim == 1:
263 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
263 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
264 else:
264 else:
265 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
265 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
266
266
267 self.dataOut.flagNoData = False
267 self.dataOut.flagNoData = False
268 self.blockIndex += 1
268 self.blockIndex += 1
269
269
270 log.log("Block No. {}/{} -> {}".format(
270 log.log("Block No. {}/{} -> {}".format(
271 self.blockIndex,
271 self.blockIndex,
272 self.blocksPerFile,
272 self.blocksPerFile,
273 self.dataOut.datatime.ctime()), self.name)
273 self.dataOut.datatime.ctime()), self.name)
274
274
275 return
275 return
276
276
277 def run(self, **kwargs):
277 def run(self, **kwargs):
278
278
279 if not(self.isConfig):
279 if not(self.isConfig):
280 self.setup(**kwargs)
280 self.setup(**kwargs)
281 self.isConfig = True
281 self.isConfig = True
282
282
283 if self.blockIndex == self.blocksPerFile:
283 if self.blockIndex == self.blocksPerFile:
284 self.setNextFile()
284 self.setNextFile()
285
285
286 self.getData()
286 self.getData()
287
287
288 return
288 return
289
289
290 @MPDecorator
290 @MPDecorator
291 class HDFWriter(Operation):
291 class HDFWriter(Operation):
292 """Operation to write HDF5 files.
292 """Operation to write HDF5 files.
293
293
294 The HDF5 file contains by default two groups Data and Metadata where
294 The HDF5 file contains by default two groups Data and Metadata where
295 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
295 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
296 parameters, data attributes are normaly time dependent where the metadata
296 parameters, data attributes are normaly time dependent where the metadata
297 are not.
297 are not.
298 It is possible to customize the structure of the HDF5 file with the
298 It is possible to customize the structure of the HDF5 file with the
299 optional description parameter see the examples.
299 optional description parameter see the examples.
300
300
301 Parameters:
301 Parameters:
302 -----------
302 -----------
303 path : str
303 path : str
304 Path where files will be saved.
304 Path where files will be saved.
305 blocksPerFile : int
305 blocksPerFile : int
306 Number of blocks per file
306 Number of blocks per file
307 metadataList : list
307 metadataList : list
308 List of the dataOut attributes that will be saved as metadata
308 List of the dataOut attributes that will be saved as metadata
309 dataList : int
309 dataList : int
310 List of the dataOut attributes that will be saved as data
310 List of the dataOut attributes that will be saved as data
311 setType : bool
311 setType : bool
312 If True the name of the files corresponds to the timestamp of the data
312 If True the name of the files corresponds to the timestamp of the data
313 description : dict, optional
313 description : dict, optional
314 Dictionary with the desired description of the HDF5 file
314 Dictionary with the desired description of the HDF5 file
315
315
316 Examples
316 Examples
317 --------
317 --------
318
318
319 desc = {
319 desc = {
320 'data_output': {'winds': ['z', 'w', 'v']},
320 'data_output': {'winds': ['z', 'w', 'v']},
321 'utctime': 'timestamps',
321 'utctime': 'timestamps',
322 'heightList': 'heights'
322 'heightList': 'heights'
323 }
323 }
324 desc = {
324 desc = {
325 'data_output': ['z', 'w', 'v'],
325 'data_output': ['z', 'w', 'v'],
326 'utctime': 'timestamps',
326 'utctime': 'timestamps',
327 'heightList': 'heights'
327 'heightList': 'heights'
328 }
328 }
329 desc = {
329 desc = {
330 'Data': {
330 'Data': {
331 'data_output': 'winds',
331 'data_output': 'winds',
332 'utctime': 'timestamps'
332 'utctime': 'timestamps'
333 },
333 },
334 'Metadata': {
334 'Metadata': {
335 'heightList': 'heights'
335 'heightList': 'heights'
336 }
336 }
337 }
337 }
338
338
339 writer = proc_unit.addOperation(name='HDFWriter')
339 writer = proc_unit.addOperation(name='HDFWriter')
340 writer.addParameter(name='path', value='/path/to/file')
340 writer.addParameter(name='path', value='/path/to/file')
341 writer.addParameter(name='blocksPerFile', value='32')
341 writer.addParameter(name='blocksPerFile', value='32')
342 writer.addParameter(name='metadataList', value='heightList,timeZone')
342 writer.addParameter(name='metadataList', value='heightList,timeZone')
343 writer.addParameter(name='dataList',value='data_output,utctime')
343 writer.addParameter(name='dataList',value='data_output,utctime')
344 # writer.addParameter(name='description',value=json.dumps(desc))
344 # writer.addParameter(name='description',value=json.dumps(desc))
345
345
346 """
346 """
347
347
348 ext = ".hdf5"
348 ext = ".hdf5"
349 optchar = "D"
349 optchar = "D"
350 filename = None
350 filename = None
351 path = None
351 path = None
352 setFile = None
352 setFile = None
353 fp = None
353 fp = None
354 firsttime = True
354 firsttime = True
355 #Configurations
355 #Configurations
356 blocksPerFile = None
356 blocksPerFile = None
357 blockIndex = None
357 blockIndex = None
358 dataOut = None
358 dataOut = None
359 #Data Arrays
359 #Data Arrays
360 dataList = None
360 dataList = None
361 metadataList = None
361 metadataList = None
362 currentDay = None
362 currentDay = None
363 lastTime = None
363 lastTime = None
364
364
365 def __init__(self):
365 def __init__(self):
366
366
367 Operation.__init__(self)
367 Operation.__init__(self)
368 return
368 return
369
369
370 def set_kwargs(self, **kwargs):
370 def set_kwargs(self, **kwargs):
371
371
372 for key, value in kwargs.items():
372 for key, value in kwargs.items():
373 setattr(self, key, value)
373 setattr(self, key, value)
374
374
375 def set_kwargs_obj(self, obj, **kwargs):
375 def set_kwargs_obj(self, obj, **kwargs):
376
376
377 for key, value in kwargs.items():
377 for key, value in kwargs.items():
378 setattr(obj, key, value)
378 setattr(obj, key, value)
379
379
380 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None, **kwargs):
380 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None, **kwargs):
381 self.path = path
381 self.path = path
382 self.blocksPerFile = blocksPerFile
382 self.blocksPerFile = blocksPerFile
383 self.metadataList = metadataList
383 self.metadataList = metadataList
384 self.dataList = [s.strip() for s in dataList]
384 self.dataList = [s.strip() for s in dataList]
385 self.setType = setType
385 self.setType = setType
386 self.description = description
386 self.description = description
387 self.set_kwargs(**kwargs)
387 self.set_kwargs(**kwargs)
388 #print("self.uniqueChannel: ", self.uniqueChannel)
388 #print("self.uniqueChannel: ", self.uniqueChannel)
389 #self.uniqueChannel = uniqueChannel
389 self.uniqueChannel = uniqueChannel
390
390
391 if self.metadataList is None:
391 if self.metadataList is None:
392 self.metadataList = self.dataOut.metadata_list
392 self.metadataList = self.dataOut.metadata_list
393
393
394 tableList = []
394 tableList = []
395 dsList = []
395 dsList = []
396
396
397 for i in range(len(self.dataList)):
397 for i in range(len(self.dataList)):
398 dsDict = {}
398 dsDict = {}
399 if hasattr(self.dataOut, self.dataList[i]):
399 if hasattr(self.dataOut, self.dataList[i]):
400 dataAux = getattr(self.dataOut, self.dataList[i])
400 dataAux = getattr(self.dataOut, self.dataList[i])
401 dsDict['variable'] = self.dataList[i]
401 dsDict['variable'] = self.dataList[i]
402 else:
402 else:
403 log.warning('Attribute {} not found in dataOut', self.name)
403 log.warning('Attribute {} not found in dataOut', self.name)
404 continue
404 continue
405
405
406 if dataAux is None:
406 if dataAux is None:
407 continue
407 continue
408 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
408 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float_)):
409 dsDict['nDim'] = 0
409 dsDict['nDim'] = 0
410 else:
410 else:
411 if self.uniqueChannel: #Creates extra dimension to avoid the creation of multiple channels
411 if self.uniqueChannel: #Creates extra dimension to avoid the creation of multiple channels
412 dataAux = numpy.expand_dims(dataAux, axis=0)
412 dataAux = numpy.expand_dims(dataAux, axis=0)
413 #setattr(self.dataOut, self.dataList[i], numpy.expand_dims(getattr(self.dataOut, self.dataList[i]), axis=0))
413 #setattr(self.dataOut, self.dataList[i], numpy.expand_dims(getattr(self.dataOut, self.dataList[i]), axis=0))
414 #dataAux = getattr(self.dataOut, self.dataList[i])
414 #dataAux = getattr(self.dataOut, self.dataList[i])
415 #print(getattr(self.dataOut, self.dataList[i]))
415 #print(getattr(self.dataOut, self.dataList[i]))
416 dsDict['nDim'] = len(dataAux.shape)
416 dsDict['nDim'] = len(dataAux.shape)
417 dsDict['shape'] = dataAux.shape
417 dsDict['shape'] = dataAux.shape
418 dsDict['dsNumber'] = dataAux.shape[0]
418 dsDict['dsNumber'] = dataAux.shape[0]
419 dsDict['dtype'] = dataAux.dtype
419 dsDict['dtype'] = dataAux.dtype
420
420
421 dsList.append(dsDict)
421 dsList.append(dsDict)
422
422
423 self.dsList = dsList
423 self.dsList = dsList
424 self.currentDay = self.dataOut.datatime.date()
424 self.currentDay = self.dataOut.datatime.date()
425
425
426 def timeFlag(self):
426 def timeFlag(self):
427 currentTime = self.dataOut.utctime
427 currentTime = self.dataOut.utctime
428 timeTuple = time.localtime(currentTime)
428 timeTuple = time.localtime(currentTime)
429 dataDay = timeTuple.tm_yday
429 dataDay = timeTuple.tm_yday
430
430
431 if self.lastTime is None:
431 if self.lastTime is None:
432 self.lastTime = currentTime
432 self.lastTime = currentTime
433 self.currentDay = dataDay
433 self.currentDay = dataDay
434 return False
434 return False
435
435
436 timeDiff = currentTime - self.lastTime
436 timeDiff = currentTime - self.lastTime
437
437
438 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
438 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
439 if dataDay != self.currentDay:
439 if dataDay != self.currentDay:
440 self.currentDay = dataDay
440 self.currentDay = dataDay
441 return True
441 return True
442 elif timeDiff > 3*60*60:
442 elif timeDiff > 3*60*60:
443 self.lastTime = currentTime
443 self.lastTime = currentTime
444 return True
444 return True
445 else:
445 else:
446 self.lastTime = currentTime
446 self.lastTime = currentTime
447 return False
447 return False
448
448
449 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
449 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
450 dataList=[], setType=None, description={}, **kwargs):
450 dataList=[], setType=None, description={}, **kwargs):
451
451
452 self.dataOut = dataOut
452 self.dataOut = dataOut
453 self.set_kwargs_obj(self.dataOut, **kwargs)
453 self.set_kwargs_obj(self.dataOut, **kwargs)
454 if not(self.isConfig):
454 if not(self.isConfig):
455 self.setup(path=path, blocksPerFile=blocksPerFile,
455 self.setup(path=path, blocksPerFile=blocksPerFile,
456 metadataList=metadataList, dataList=dataList,
456 metadataList=metadataList, dataList=dataList,
457 setType=setType, description=description, **kwargs)
457 setType=setType, description=description, **kwargs)
458
458
459 self.isConfig = True
459 self.isConfig = True
460 self.setNextFile()
460 self.setNextFile()
461
461
462 self.putData()
462 self.putData()
463
463
464 return
464 return
465
465
466 def setNextFile(self):
466 def setNextFile(self):
467
467
468 ext = self.ext
468 ext = self.ext
469 path = self.path
469 path = self.path
470 setFile = self.setFile
470 setFile = self.setFile
471
471
472 timeTuple = time.localtime(self.dataOut.utctime)
472 timeTuple = time.localtime(self.dataOut.utctime)
473 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
473 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
474 fullpath = os.path.join(path, subfolder)
474 fullpath = os.path.join(path, subfolder)
475
475
476 if os.path.exists(fullpath):
476 if os.path.exists(fullpath):
477 filesList = os.listdir(fullpath)
477 filesList = os.listdir(fullpath)
478 filesList = [k for k in filesList if k.startswith(self.optchar)]
478 filesList = [k for k in filesList if k.startswith(self.optchar)]
479 if len( filesList ) > 0:
479 if len( filesList ) > 0:
480 filesList = sorted(filesList, key=str.lower)
480 filesList = sorted(filesList, key=str.lower)
481 filen = filesList[-1]
481 filen = filesList[-1]
482 # el filename debera tener el siguiente formato
482 # el filename debera tener el siguiente formato
483 # 0 1234 567 89A BCDE (hex)
483 # 0 1234 567 89A BCDE (hex)
484 # x YYYY DDD SSS .ext
484 # x YYYY DDD SSS .ext
485 if isNumber(filen[8:11]):
485 if isNumber(filen[8:11]):
486 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
486 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
487 else:
487 else:
488 setFile = -1
488 setFile = -1
489 else:
489 else:
490 setFile = -1 #inicializo mi contador de seteo
490 setFile = -1 #inicializo mi contador de seteo
491 else:
491 else:
492 os.makedirs(fullpath)
492 os.makedirs(fullpath)
493 setFile = -1 #inicializo mi contador de seteo
493 setFile = -1 #inicializo mi contador de seteo
494
494
495 if self.setType is None:
495 if self.setType is None:
496 setFile += 1
496 setFile += 1
497 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
497 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
498 timeTuple.tm_year,
498 timeTuple.tm_year,
499 timeTuple.tm_yday,
499 timeTuple.tm_yday,
500 setFile,
500 setFile,
501 ext )
501 ext )
502 else:
502 else:
503 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
503 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
504 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
504 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
505 timeTuple.tm_year,
505 timeTuple.tm_year,
506 timeTuple.tm_yday,
506 timeTuple.tm_yday,
507 setFile,
507 setFile,
508 ext )
508 ext )
509
509
510 self.filename = os.path.join( path, subfolder, file )
510 self.filename = os.path.join( path, subfolder, file )
511
511
512 #Setting HDF5 File
512 #Setting HDF5 File
513 self.fp = h5py.File(self.filename, 'w')
513 self.fp = h5py.File(self.filename, 'w')
514 #write metadata
514 #write metadata
515 self.writeMetadata(self.fp)
515 self.writeMetadata(self.fp)
516 #Write data
516 #Write data
517 self.writeData(self.fp)
517 self.writeData(self.fp)
518
518
519 def getLabel(self, name, x=None):
519 def getLabel(self, name, x=None):
520 #print("x: ", x)
520 #print("x: ", x)
521 if x is None:
521 if x is None:
522 if 'Data' in self.description:
522 if 'Data' in self.description:
523 data = self.description['Data']
523 data = self.description['Data']
524 if 'Metadata' in self.description:
524 if 'Metadata' in self.description:
525 data.update(self.description['Metadata'])
525 data.update(self.description['Metadata'])
526 else:
526 else:
527 data = self.description
527 data = self.description
528 if name in data:
528 if name in data:
529 if isinstance(data[name], str):
529 if isinstance(data[name], str):
530 return data[name]
530 return data[name]
531 elif isinstance(data[name], list):
531 elif isinstance(data[name], list):
532 return None
532 return None
533 elif isinstance(data[name], dict):
533 elif isinstance(data[name], dict):
534 for key, value in data[name].items():
534 for key, value in data[name].items():
535 return key
535 return key
536 return name
536 return name
537 else:
537 else:
538 if 'Data' in self.description:
538 if 'Data' in self.description:
539 data = self.description['Data']
539 data = self.description['Data']
540 if 'Metadata' in self.description:
540 if 'Metadata' in self.description:
541 data.update(self.description['Metadata'])
541 data.update(self.description['Metadata'])
542 else:
542 else:
543 data = self.description
543 data = self.description
544 if name in data:
544 if name in data:
545 if isinstance(data[name], list):
545 if isinstance(data[name], list):
546 return data[name][x]
546 return data[name][x]
547 elif isinstance(data[name], dict):
547 elif isinstance(data[name], dict):
548 for key, value in data[name].items():
548 for key, value in data[name].items():
549 return value[x]
549 return value[x]
550 if 'cspc' in name:
550 if 'cspc' in name:
551 return 'pair{:02d}'.format(x)
551 return 'pair{:02d}'.format(x)
552 else:
552 else:
553 return 'channel{:02d}'.format(x)
553 return 'channel{:02d}'.format(x)
554
554
555 def writeMetadata(self, fp):
555 def writeMetadata(self, fp):
556
556
557 if self.description:
557 if self.description:
558 if 'Metadata' in self.description:
558 if 'Metadata' in self.description:
559 grp = fp.create_group('Metadata')
559 grp = fp.create_group('Metadata')
560 else:
560 else:
561 grp = fp
561 grp = fp
562 else:
562 else:
563 grp = fp.create_group('Metadata')
563 grp = fp.create_group('Metadata')
564
564
565 for i in range(len(self.metadataList)):
565 for i in range(len(self.metadataList)):
566 if not hasattr(self.dataOut, self.metadataList[i]):
566 if not hasattr(self.dataOut, self.metadataList[i]):
567 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
567 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
568 continue
568 continue
569 value = getattr(self.dataOut, self.metadataList[i])
569 value = getattr(self.dataOut, self.metadataList[i])
570 if isinstance(value, bool):
570 if isinstance(value, bool):
571 if value is True:
571 if value is True:
572 value = 1
572 value = 1
573 else:
573 else:
574 value = 0
574 value = 0
575 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
575 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
576 return
576 return
577
577
578 def writeData(self, fp):
578 def writeData(self, fp):
579
579
580 if self.description:
580 if self.description:
581 if 'Data' in self.description:
581 if 'Data' in self.description:
582 grp = fp.create_group('Data')
582 grp = fp.create_group('Data')
583 else:
583 else:
584 grp = fp
584 grp = fp
585 else:
585 else:
586 grp = fp.create_group('Data')
586 grp = fp.create_group('Data')
587
587
588 dtsets = []
588 dtsets = []
589 data = []
589 data = []
590 #print("self.dsList: ", self.dsList)
590 #print("self.dsList: ", self.dsList)
591 for dsInfo in self.dsList:
591 for dsInfo in self.dsList:
592 if dsInfo['nDim'] == 0:
592 if dsInfo['nDim'] == 0:
593 ds = grp.create_dataset(
593 ds = grp.create_dataset(
594 self.getLabel(dsInfo['variable']),
594 self.getLabel(dsInfo['variable']),
595 (self.blocksPerFile, ),
595 (self.blocksPerFile, ),
596 chunks=True,
596 chunks=True,
597 dtype=numpy.float64)
597 dtype=numpy.float64)
598 dtsets.append(ds)
598 dtsets.append(ds)
599 data.append((dsInfo['variable'], -1))
599 data.append((dsInfo['variable'], -1))
600 else:
600 else:
601 label = self.getLabel(dsInfo['variable'])
601 label = self.getLabel(dsInfo['variable'])
602 if label is not None:
602 if label is not None:
603 sgrp = grp.create_group(label)
603 sgrp = grp.create_group(label)
604 else:
604 else:
605 sgrp = grp
605 sgrp = grp
606 if self.uniqueChannel: #Creates extra dimension to avoid the creation of multiple channels
606 if self.uniqueChannel: #Creates extra dimension to avoid the creation of multiple channels
607 setattr(self.dataOut, dsInfo['variable'], numpy.expand_dims(getattr(self.dataOut, dsInfo['variable']), axis=0))
607 setattr(self.dataOut, dsInfo['variable'], numpy.expand_dims(getattr(self.dataOut, dsInfo['variable']), axis=0))
608 for i in range(dsInfo['dsNumber']):
608 for i in range(dsInfo['dsNumber']):
609 ds = sgrp.create_dataset(
609 ds = sgrp.create_dataset(
610 self.getLabel(dsInfo['variable'], i),
610 self.getLabel(dsInfo['variable'], i),
611 (self.blocksPerFile, ) + dsInfo['shape'][1:],
611 (self.blocksPerFile, ) + dsInfo['shape'][1:],
612 chunks=True,
612 chunks=True,
613 dtype=dsInfo['dtype'])
613 dtype=dsInfo['dtype'])
614 dtsets.append(ds)
614 dtsets.append(ds)
615 data.append((dsInfo['variable'], i))
615 data.append((dsInfo['variable'], i))
616
616
617 fp.flush()
617 fp.flush()
618
618
619 log.log('Creating file: {}'.format(fp.filename), self.name)
619 log.log('Creating file: {}'.format(fp.filename), self.name)
620
620
621 self.ds = dtsets
621 self.ds = dtsets
622 self.data = data
622 self.data = data
623 self.firsttime = True
623 self.firsttime = True
624 self.blockIndex = 0
624 self.blockIndex = 0
625 return
625 return
626
626
627 def putData(self):
627 def putData(self):
628
628
629 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
629 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
630 self.closeFile()
630 self.closeFile()
631 self.setNextFile()
631 self.setNextFile()
632
632
633 for i, ds in enumerate(self.ds):
633 for i, ds in enumerate(self.ds):
634 attr, ch = self.data[i]
634 attr, ch = self.data[i]
635 if ch == -1:
635 if ch == -1:
636 ds[self.blockIndex] = getattr(self.dataOut, attr)
636 ds[self.blockIndex] = getattr(self.dataOut, attr)
637 else:
637 else:
638 if self.uniqueChannel and self.blockIndex != 0: #Creates extra dimension to avoid the creation of multiple channels
638 if self.uniqueChannel and self.blockIndex != 0: #Creates extra dimension to avoid the creation of multiple channels
639 setattr(self.dataOut, attr, numpy.expand_dims(getattr(self.dataOut, attr), axis=0))
639 setattr(self.dataOut, attr, numpy.expand_dims(getattr(self.dataOut, attr), axis=0))
640 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
640 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
641 if self.uniqueChannel: #Deletes extra dimension created to avoid the creation of multiple channels
641 if self.uniqueChannel: #Deletes extra dimension created to avoid the creation of multiple channels
642 setattr(self.dataOut, attr, getattr(self.dataOut, attr)[0])
642 setattr(self.dataOut, attr, getattr(self.dataOut, attr)[0])
643
643
644 self.fp.flush()
644 self.fp.flush()
645 self.blockIndex += 1
645 self.blockIndex += 1
646 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
646 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
647
647
648 return
648 return
649
649
650 def closeFile(self):
650 def closeFile(self):
651
651
652 if self.blockIndex != self.blocksPerFile:
652 if self.blockIndex != self.blocksPerFile:
653 for ds in self.ds:
653 for ds in self.ds:
654 ds.resize(self.blockIndex, axis=0)
654 ds.resize(self.blockIndex, axis=0)
655
655
656 if self.fp:
656 if self.fp:
657 self.fp.flush()
657 self.fp.flush()
658 self.fp.close()
658 self.fp.close()
659
659
660 def close(self):
660 def close(self):
661
661
662 self.closeFile()
662 self.closeFile()
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
General Comments 0
You need to be logged in to leave comments. Login now