##// END OF EJS Templates
schain
RSS
cambio a timeZone
joabAM -
r1402:2b5adcd8c887
parent child
Show More
Side by Side Unified
Context file:
r1402:2b5adcd8c887 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,674 +1,674
1 import os
1 import os
2 import time
2 import time
3 import datetime
3 import datetime
4
4
5 import numpy
5 import numpy
6 import h5py
6 import h5py
7
7
8 import schainpy.admin
8 import schainpy.admin
9 from schainpy.model.data.jrodata import *
9 from schainpy.model.data.jrodata import *
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 from schainpy.utils import log
12 from schainpy.utils import log
13
13
14
14
15 class HDFReader(Reader, ProcessingUnit):
15 class HDFReader(Reader, ProcessingUnit):
16 """Processing unit to read HDF5 format files
16 """Processing unit to read HDF5 format files
17
17
18 This unit reads HDF5 files created with `HDFWriter` operation contains
18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 attributes.
20 attributes.
21 It is possible to read any HDF5 file by given the structure in the `description`
21 It is possible to read any HDF5 file by given the structure in the `description`
22 parameter, also you can add extra values to metadata with the parameter `extras`.
22 parameter, also you can add extra values to metadata with the parameter `extras`.
23
23
24 Parameters:
24 Parameters:
25 -----------
25 -----------
26 path : str
26 path : str
27 Path where files are located.
27 Path where files are located.
28 startDate : date
28 startDate : date
29 Start date of the files
29 Start date of the files
30 endDate : list
30 endDate : list
31 End date of the files
31 End date of the files
32 startTime : time
32 startTime : time
33 Start time of the files
33 Start time of the files
34 endTime : time
34 endTime : time
35 End time of the files
35 End time of the files
36 description : dict, optional
36 description : dict, optional
37 Dictionary with the description of the HDF5 file
37 Dictionary with the description of the HDF5 file
38 extras : dict, optional
38 extras : dict, optional
39 Dictionary with extra metadata to be be added to `dataOut`
39 Dictionary with extra metadata to be be added to `dataOut`
40
40
41 Examples
41 Examples
42 --------
42 --------
43
43
44 desc = {
44 desc = {
45 'Data': {
45 'Data': {
46 'data_output': ['u', 'v', 'w'],
46 'data_output': ['u', 'v', 'w'],
47 'utctime': 'timestamps',
47 'utctime': 'timestamps',
48 } ,
48 } ,
49 'Metadata': {
49 'Metadata': {
50 'heightList': 'heights'
50 'heightList': 'heights'
51 }
51 }
52 }
52 }
53
53
54 desc = {
54 desc = {
55 'Data': {
55 'Data': {
56 'data_output': 'winds',
56 'data_output': 'winds',
57 'utctime': 'timestamps'
57 'utctime': 'timestamps'
58 },
58 },
59 'Metadata': {
59 'Metadata': {
60 'heightList': 'heights'
60 'heightList': 'heights'
61 }
61 }
62 }
62 }
63
63
64 extras = {
64 extras = {
65 'timeZone': 300
65 'timeZone': 300
66 }
66 }
67
67
68 reader = project.addReadUnit(
68 reader = project.addReadUnit(
69 name='HDFReader',
69 name='HDFReader',
70 path='/path/to/files',
70 path='/path/to/files',
71 startDate='2019/01/01',
71 startDate='2019/01/01',
72 endDate='2019/01/31',
72 endDate='2019/01/31',
73 startTime='00:00:00',
73 startTime='00:00:00',
74 endTime='23:59:59',
74 endTime='23:59:59',
75 # description=json.dumps(desc),
75 # description=json.dumps(desc),
76 # extras=json.dumps(extras),
76 # extras=json.dumps(extras),
77 )
77 )
78
78
79 """
79 """
80
80
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82
82
83 def __init__(self):
83 def __init__(self):
84 ProcessingUnit.__init__(self)
84 ProcessingUnit.__init__(self)
85 self.dataOut = Parameters()
85 self.dataOut = Parameters()
86 self.ext = ".hdf5"
86 self.ext = ".hdf5"
87 self.optchar = "D"
87 self.optchar = "D"
88 self.meta = {}
88 self.meta = {}
89 self.data = {}
89 self.data = {}
90 self.open_file = h5py.File
90 self.open_file = h5py.File
91 self.open_mode = 'r'
91 self.open_mode = 'r'
92 self.description = {}
92 self.description = {}
93 self.extras = {}
93 self.extras = {}
94 self.filefmt = "*%Y%j***"
94 self.filefmt = "*%Y%j***"
95 self.folderfmt = "*%Y%j"
95 self.folderfmt = "*%Y%j"
96 self.utcoffset = 0
96 self.utcoffset = 0
97
97
98 def setup(self, **kwargs):
98 def setup(self, **kwargs):
99
99
100 self.set_kwargs(**kwargs)
100 self.set_kwargs(**kwargs)
101 if not self.ext.startswith('.'):
101 if not self.ext.startswith('.'):
102 self.ext = '.{}'.format(self.ext)
102 self.ext = '.{}'.format(self.ext)
103
103
104 if self.online:
104 if self.online:
105 log.log("Searching files in online mode...", self.name)
105 log.log("Searching files in online mode...", self.name)
106
106
107 for nTries in range(self.nTries):
107 for nTries in range(self.nTries):
108 fullpath = self.searchFilesOnLine(self.path, self.startDate,
108 fullpath = self.searchFilesOnLine(self.path, self.startDate,
109 self.endDate, self.expLabel, self.ext, self.walk,
109 self.endDate, self.expLabel, self.ext, self.walk,
110 self.filefmt, self.folderfmt)
110 self.filefmt, self.folderfmt)
111 pathname, filename = os.path.split(fullpath)
111 pathname, filename = os.path.split(fullpath)
112 #print(pathname,filename)
112 #print(pathname,filename)
113 try:
113 try:
114 fullpath = next(fullpath)
114 fullpath = next(fullpath)
115
115
116 except:
116 except:
117 fullpath = None
117 fullpath = None
118
118
119 if fullpath:
119 if fullpath:
120 break
120 break
121
121
122 log.warning(
122 log.warning(
123 'Waiting {} sec for a valid file in {}: try {} ...'.format(
123 'Waiting {} sec for a valid file in {}: try {} ...'.format(
124 self.delay, self.path, nTries + 1),
124 self.delay, self.path, nTries + 1),
125 self.name)
125 self.name)
126 time.sleep(self.delay)
126 time.sleep(self.delay)
127
127
128 if not(fullpath):
128 if not(fullpath):
129 raise schainpy.admin.SchainError(
129 raise schainpy.admin.SchainError(
130 'There isn\'t any valid file in {}'.format(self.path))
130 'There isn\'t any valid file in {}'.format(self.path))
131
131
132 pathname, filename = os.path.split(fullpath)
132 pathname, filename = os.path.split(fullpath)
133 self.year = int(filename[1:5])
133 self.year = int(filename[1:5])
134 self.doy = int(filename[5:8])
134 self.doy = int(filename[5:8])
135 self.set = int(filename[8:11]) - 1
135 self.set = int(filename[8:11]) - 1
136 else:
136 else:
137 log.log("Searching files in {}".format(self.path), self.name)
137 log.log("Searching files in {}".format(self.path), self.name)
138 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
138 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
139 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
139 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
140
140
141 self.setNextFile()
141 self.setNextFile()
142
142
143 return
143 return
144
144
145
145
146 def readFirstHeader(self):
146 def readFirstHeader(self):
147 '''Read metadata and data'''
147 '''Read metadata and data'''
148
148
149 self.__readMetadata()
149 self.__readMetadata()
150 self.__readData()
150 self.__readData()
151 self.__setBlockList()
151 self.__setBlockList()
152
152
153 if 'type' in self.meta:
153 if 'type' in self.meta:
154 ##print("Creting dataOut...")
154 ##print("Creting dataOut...")
155 self.dataOut = eval(self.meta['type'])()
155 self.dataOut = eval(self.meta['type'])()
156 ##print(vars(self.dataOut))
156 ##print(vars(self.dataOut))
157
157
158 for attr in self.meta:
158 for attr in self.meta:
159 ##print("attr: ", attr)
159 ##print("attr: ", attr)
160 ##print(type(self.dataOut).__name__)
160 ##print(type(self.dataOut).__name__)
161 setattr(self.dataOut, attr, self.meta[attr])
161 setattr(self.dataOut, attr, self.meta[attr])
162
162
163 self.blockIndex = 0
163 self.blockIndex = 0
164
164
165 return
165 return
166
166
167 def __setBlockList(self):
167 def __setBlockList(self):
168 '''
168 '''
169 Selects the data within the times defined
169 Selects the data within the times defined
170
170
171 self.fp
171 self.fp
172 self.startTime
172 self.startTime
173 self.endTime
173 self.endTime
174 self.blockList
174 self.blockList
175 self.blocksPerFile
175 self.blocksPerFile
176
176
177 '''
177 '''
178
178
179 startTime = self.startTime
179 startTime = self.startTime
180 endTime = self.endTime
180 endTime = self.endTime
181 thisUtcTime = self.data['utctime'] + self.utcoffset
181 thisUtcTime = self.data['utctime'] + self.utcoffset
182 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
182 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
183 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
183 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
184 self.startFileDatetime = thisDatetime
184 self.startFileDatetime = thisDatetime
185 thisDate = thisDatetime.date()
185 thisDate = thisDatetime.date()
186 thisTime = thisDatetime.time()
186 thisTime = thisDatetime.time()
187
187
188 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
188 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
189 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
189 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
190
190
191 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
191 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
192
192
193 self.blockList = ind
193 self.blockList = ind
194 self.blocksPerFile = len(ind)
194 self.blocksPerFile = len(ind)
195 self.blocksPerFile = len(thisUtcTime)
195 self.blocksPerFile = len(thisUtcTime)
196 return
196 return
197
197
198 def __readMetadata(self):
198 def __readMetadata(self):
199 '''
199 '''
200 Reads Metadata
200 Reads Metadata
201 '''
201 '''
202
202
203 meta = {}
203 meta = {}
204
204
205 if self.description:
205 if self.description:
206 for key, value in self.description['Metadata'].items():
206 for key, value in self.description['Metadata'].items():
207 meta[key] = self.fp[value][()]
207 meta[key] = self.fp[value][()]
208 else:
208 else:
209 grp = self.fp['Metadata']
209 grp = self.fp['Metadata']
210 for name in grp:
210 for name in grp:
211 meta[name] = grp[name][()]
211 meta[name] = grp[name][()]
212
212
213 if self.extras:
213 if self.extras:
214 for key, value in self.extras.items():
214 for key, value in self.extras.items():
215 meta[key] = value
215 meta[key] = value
216 self.meta = meta
216 self.meta = meta
217
217
218 return
218 return
219
219
220
220
221
221
222 def checkForRealPath(self, nextFile, nextDay):
222 def checkForRealPath(self, nextFile, nextDay):
223
223
224 # print("check FRP")
224 # print("check FRP")
225 # dt = self.startFileDatetime + datetime.timedelta(1)
225 # dt = self.startFileDatetime + datetime.timedelta(1)
226 # filename = '{}.{}{}'.format(self.path, dt.strftime('%Y%m%d'), self.ext)
226 # filename = '{}.{}{}'.format(self.path, dt.strftime('%Y%m%d'), self.ext)
227 # fullfilename = os.path.join(self.path, filename)
227 # fullfilename = os.path.join(self.path, filename)
228 # print("check Path ",fullfilename,filename)
228 # print("check Path ",fullfilename,filename)
229 # if os.path.exists(fullfilename):
229 # if os.path.exists(fullfilename):
230 # return fullfilename, filename
230 # return fullfilename, filename
231 # return None, filename
231 # return None, filename
232 return None,None
232 return None,None
233
233
234 def __readData(self):
234 def __readData(self):
235
235
236 data = {}
236 data = {}
237
237
238 if self.description:
238 if self.description:
239 for key, value in self.description['Data'].items():
239 for key, value in self.description['Data'].items():
240 if isinstance(value, str):
240 if isinstance(value, str):
241 if isinstance(self.fp[value], h5py.Dataset):
241 if isinstance(self.fp[value], h5py.Dataset):
242 data[key] = self.fp[value][()]
242 data[key] = self.fp[value][()]
243 elif isinstance(self.fp[value], h5py.Group):
243 elif isinstance(self.fp[value], h5py.Group):
244 array = []
244 array = []
245 for ch in self.fp[value]:
245 for ch in self.fp[value]:
246 array.append(self.fp[value][ch][()])
246 array.append(self.fp[value][ch][()])
247 data[key] = numpy.array(array)
247 data[key] = numpy.array(array)
248 elif isinstance(value, list):
248 elif isinstance(value, list):
249 array = []
249 array = []
250 for ch in value:
250 for ch in value:
251 array.append(self.fp[ch][()])
251 array.append(self.fp[ch][()])
252 data[key] = numpy.array(array)
252 data[key] = numpy.array(array)
253 else:
253 else:
254 grp = self.fp['Data']
254 grp = self.fp['Data']
255 for name in grp:
255 for name in grp:
256 if isinstance(grp[name], h5py.Dataset):
256 if isinstance(grp[name], h5py.Dataset):
257 array = grp[name][()]
257 array = grp[name][()]
258 elif isinstance(grp[name], h5py.Group):
258 elif isinstance(grp[name], h5py.Group):
259 array = []
259 array = []
260 for ch in grp[name]:
260 for ch in grp[name]:
261 array.append(grp[name][ch][()])
261 array.append(grp[name][ch][()])
262 array = numpy.array(array)
262 array = numpy.array(array)
263 else:
263 else:
264 log.warning('Unknown type: {}'.format(name))
264 log.warning('Unknown type: {}'.format(name))
265
265
266 if name in self.description:
266 if name in self.description:
267 key = self.description[name]
267 key = self.description[name]
268 else:
268 else:
269 key = name
269 key = name
270 data[key] = array
270 data[key] = array
271
271
272 self.data = data
272 self.data = data
273 return
273 return
274
274
275 def getData(self):
275 def getData(self):
276 if not self.isDateTimeInRange(self.startFileDatetime, self.startDate, self.endDate, self.startTime, self.endTime):
276 if not self.isDateTimeInRange(self.startFileDatetime, self.startDate, self.endDate, self.startTime, self.endTime):
277 self.dataOut.flagNoData = True
277 self.dataOut.flagNoData = True
278 self.blockIndex = self.blocksPerFile
278 self.blockIndex = self.blocksPerFile
279 #self.dataOut.error = True TERMINA EL PROGRAMA, removido
279 #self.dataOut.error = True TERMINA EL PROGRAMA, removido
280 return
280 return
281 for attr in self.data:
281 for attr in self.data:
282 #print("attr ",attr)
282 #print("attr ",attr)
283 if self.data[attr].ndim == 1:
283 if self.data[attr].ndim == 1:
284 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
284 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
285 else:
285 else:
286 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
286 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
287
287
288
288
289 self.blockIndex += 1
289 self.blockIndex += 1
290
290
291 if self.blockIndex == 1:
291 if self.blockIndex == 1:
292 log.log("Block No. {}/{} -> {}".format(
292 log.log("Block No. {}/{} -> {}".format(
293 self.blockIndex,
293 self.blockIndex,
294 self.blocksPerFile,
294 self.blocksPerFile,
295 self.dataOut.datatime.ctime()), self.name)
295 self.dataOut.datatime.ctime()), self.name)
296 else:
296 else:
297 log.log("Block No. {}/{} ".format(
297 log.log("Block No. {}/{} ".format(
298 self.blockIndex,
298 self.blockIndex,
299 self.blocksPerFile),self.name)
299 self.blocksPerFile),self.name)
300
300
301 self.dataOut.flagNoData = False
301 self.dataOut.flagNoData = False
302 self.dataOut.error = False
302 self.dataOut.error = False
303 return
303 return
304
304
305 def run(self, **kwargs):
305 def run(self, **kwargs):
306
306
307 if not(self.isConfig):
307 if not(self.isConfig):
308 self.setup(**kwargs)
308 self.setup(**kwargs)
309 self.isConfig = True
309 self.isConfig = True
310
310
311 if self.blockIndex == self.blocksPerFile:
311 if self.blockIndex == self.blocksPerFile:
312 self.setNextFile()
312 self.setNextFile()
313
313
314 self.getData()
314 self.getData()
315
315
316 return
316 return
317
317
318 @MPDecorator
318 @MPDecorator
319 class HDFWriter(Operation):
319 class HDFWriter(Operation):
320 """Operation to write HDF5 files.
320 """Operation to write HDF5 files.
321
321
322 The HDF5 file contains by default two groups Data and Metadata where
322 The HDF5 file contains by default two groups Data and Metadata where
323 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
323 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
324 parameters, data attributes are normaly time dependent where the metadata
324 parameters, data attributes are normaly time dependent where the metadata
325 are not.
325 are not.
326 It is possible to customize the structure of the HDF5 file with the
326 It is possible to customize the structure of the HDF5 file with the
327 optional description parameter see the examples.
327 optional description parameter see the examples.
328
328
329 Parameters:
329 Parameters:
330 -----------
330 -----------
331 path : str
331 path : str
332 Path where files will be saved.
332 Path where files will be saved.
333 blocksPerFile : int
333 blocksPerFile : int
334 Number of blocks per file
334 Number of blocks per file
335 metadataList : list
335 metadataList : list
336 List of the dataOut attributes that will be saved as metadata
336 List of the dataOut attributes that will be saved as metadata
337 dataList : int
337 dataList : int
338 List of the dataOut attributes that will be saved as data
338 List of the dataOut attributes that will be saved as data
339 setType : bool
339 setType : bool
340 If True the name of the files corresponds to the timestamp of the data
340 If True the name of the files corresponds to the timestamp of the data
341 description : dict, optional
341 description : dict, optional
342 Dictionary with the desired description of the HDF5 file
342 Dictionary with the desired description of the HDF5 file
343
343
344 Examples
344 Examples
345 --------
345 --------
346
346
347 desc = {
347 desc = {
348 'data_output': {'winds': ['z', 'w', 'v']},
348 'data_output': {'winds': ['z', 'w', 'v']},
349 'utctime': 'timestamps',
349 'utctime': 'timestamps',
350 'heightList': 'heights'
350 'heightList': 'heights'
351 }
351 }
352 desc = {
352 desc = {
353 'data_output': ['z', 'w', 'v'],
353 'data_output': ['z', 'w', 'v'],
354 'utctime': 'timestamps',
354 'utctime': 'timestamps',
355 'heightList': 'heights'
355 'heightList': 'heights'
356 }
356 }
357 desc = {
357 desc = {
358 'Data': {
358 'Data': {
359 'data_output': 'winds',
359 'data_output': 'winds',
360 'utctime': 'timestamps'
360 'utctime': 'timestamps'
361 },
361 },
362 'Metadata': {
362 'Metadata': {
363 'heightList': 'heights'
363 'heightList': 'heights'
364 }
364 }
365 }
365 }
366
366
367 writer = proc_unit.addOperation(name='HDFWriter')
367 writer = proc_unit.addOperation(name='HDFWriter')
368 writer.addParameter(name='path', value='/path/to/file')
368 writer.addParameter(name='path', value='/path/to/file')
369 writer.addParameter(name='blocksPerFile', value='32')
369 writer.addParameter(name='blocksPerFile', value='32')
370 writer.addParameter(name='metadataList', value='heightList,timeZone')
370 writer.addParameter(name='metadataList', value='heightList,timeZone')
371 writer.addParameter(name='dataList',value='data_output,utctime')
371 writer.addParameter(name='dataList',value='data_output,utctime')
372 # writer.addParameter(name='description',value=json.dumps(desc))
372 # writer.addParameter(name='description',value=json.dumps(desc))
373
373
374 """
374 """
375
375
376 ext = ".hdf5"
376 ext = ".hdf5"
377 optchar = "D"
377 optchar = "D"
378 filename = None
378 filename = None
379 path = None
379 path = None
380 setFile = None
380 setFile = None
381 fp = None
381 fp = None
382 firsttime = True
382 firsttime = True
383 #Configurations
383 #Configurations
384 blocksPerFile = None
384 blocksPerFile = None
385 blockIndex = None
385 blockIndex = None
386 dataOut = None
386 dataOut = None
387 #Data Arrays
387 #Data Arrays
388 dataList = None
388 dataList = None
389 metadataList = None
389 metadataList = None
390 currentDay = None
390 currentDay = None
391 lastTime = None
391 lastTime = None
392 typeTime = "ut"
392 timeZone = "ut"
393 hourLimit = 3
393 hourLimit = 3
394 breakDays = True
394 breakDays = True
395
395
396 def __init__(self):
396 def __init__(self):
397
397
398 Operation.__init__(self)
398 Operation.__init__(self)
399 return
399 return
400
400
401 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None,
401 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None,
402 description=None,typeTime = "ut",hourLimit = 3, breakDays=True):
402 description=None,timeZone = "ut",hourLimit = 3, breakDays=True):
403 self.path = path
403 self.path = path
404 self.blocksPerFile = blocksPerFile
404 self.blocksPerFile = blocksPerFile
405 self.metadataList = metadataList
405 self.metadataList = metadataList
406 self.dataList = [s.strip() for s in dataList]
406 self.dataList = [s.strip() for s in dataList]
407 self.setType = setType
407 self.setType = setType
408 self.description = description
408 self.description = description
409 self.timeZone = typeTime
409 self.timeZone = timeZone
410 self.hourLimit = hourLimit
410 self.hourLimit = hourLimit
411 self.breakDays = breakDays
411 self.breakDays = breakDays
412
412
413 if self.metadataList is None:
413 if self.metadataList is None:
414 self.metadataList = self.dataOut.metadata_list
414 self.metadataList = self.dataOut.metadata_list
415
415
416 tableList = []
416 tableList = []
417 dsList = []
417 dsList = []
418
418
419 for i in range(len(self.dataList)):
419 for i in range(len(self.dataList)):
420 dsDict = {}
420 dsDict = {}
421 if hasattr(self.dataOut, self.dataList[i]):
421 if hasattr(self.dataOut, self.dataList[i]):
422 dataAux = getattr(self.dataOut, self.dataList[i])
422 dataAux = getattr(self.dataOut, self.dataList[i])
423 dsDict['variable'] = self.dataList[i]
423 dsDict['variable'] = self.dataList[i]
424 else:
424 else:
425 log.warning('Attribute {} not found in dataOut', self.name)
425 log.warning('Attribute {} not found in dataOut', self.name)
426 continue
426 continue
427
427
428 if dataAux is None:
428 if dataAux is None:
429 continue
429 continue
430 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
430 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
431 dsDict['nDim'] = 0
431 dsDict['nDim'] = 0
432 else:
432 else:
433 dsDict['nDim'] = len(dataAux.shape)
433 dsDict['nDim'] = len(dataAux.shape)
434 dsDict['shape'] = dataAux.shape
434 dsDict['shape'] = dataAux.shape
435 dsDict['dsNumber'] = dataAux.shape[0]
435 dsDict['dsNumber'] = dataAux.shape[0]
436 dsDict['dtype'] = dataAux.dtype
436 dsDict['dtype'] = dataAux.dtype
437
437
438 dsList.append(dsDict)
438 dsList.append(dsDict)
439
439
440 self.dsList = dsList
440 self.dsList = dsList
441 self.currentDay = self.dataOut.datatime.date()
441 self.currentDay = self.dataOut.datatime.date()
442
442
443 def timeFlag(self):
443 def timeFlag(self):
444 currentTime = self.dataOut.utctime
444 currentTime = self.dataOut.utctime
445 if self.timeZone == "lt":
445 if self.timeZone == "lt":
446 timeTuple = time.localtime(currentTime)
446 timeTuple = time.localtime(currentTime)
447 elif self.timeZone == "ut":
447 elif self.timeZone == "ut":
448 timeTuple = time.gmtime(currentTime)
448 timeTuple = time.gmtime(currentTime)
449
449
450 dataDay = timeTuple.tm_yday
450 dataDay = timeTuple.tm_yday
451 #print("time UTC: ",currentTime, self.dataOut.datatime)
451 #print("time UTC: ",currentTime, self.dataOut.datatime)
452 if self.lastTime is None:
452 if self.lastTime is None:
453 self.lastTime = currentTime
453 self.lastTime = currentTime
454 self.currentDay = dataDay
454 self.currentDay = dataDay
455 return False
455 return False
456
456
457 timeDiff = currentTime - self.lastTime
457 timeDiff = currentTime - self.lastTime
458
458
459 #Si el dia es diferente o si la diferencia entre un dato y otro supera self.hourLimit
459 #Si el dia es diferente o si la diferencia entre un dato y otro supera self.hourLimit
460 if (dataDay != self.currentDay) and self.breakDays:
460 if (dataDay != self.currentDay) and self.breakDays:
461 self.currentDay = dataDay
461 self.currentDay = dataDay
462 return True
462 return True
463 elif timeDiff > self.hourLimit*60*60:
463 elif timeDiff > self.hourLimit*60*60:
464 self.lastTime = currentTime
464 self.lastTime = currentTime
465 return True
465 return True
466 else:
466 else:
467 self.lastTime = currentTime
467 self.lastTime = currentTime
468 return False
468 return False
469
469
470 def run(self, dataOut,**kwargs):
470 def run(self, dataOut,**kwargs):
471
471
472 self.dataOut = dataOut
472 self.dataOut = dataOut
473 if not(self.isConfig):
473 if not(self.isConfig):
474 self.setup(**kwargs)
474 self.setup(**kwargs)
475
475
476 self.isConfig = True
476 self.isConfig = True
477 self.setNextFile()
477 self.setNextFile()
478
478
479 self.putData()
479 self.putData()
480 return
480 return
481
481
482 def setNextFile(self):
482 def setNextFile(self):
483
483
484 ext = self.ext
484 ext = self.ext
485 path = self.path
485 path = self.path
486 setFile = self.setFile
486 setFile = self.setFile
487 if self.timeZone == "lt":
487 if self.timeZone == "lt":
488 timeTuple = time.localtime(self.dataOut.utctime)
488 timeTuple = time.localtime(self.dataOut.utctime)
489 elif self.timeZone == "ut":
489 elif self.timeZone == "ut":
490 timeTuple = time.gmtime(self.dataOut.utctime)
490 timeTuple = time.gmtime(self.dataOut.utctime)
491 #print("path: ",timeTuple)
491 #print("path: ",timeTuple)
492 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
492 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
493 fullpath = os.path.join(path, subfolder)
493 fullpath = os.path.join(path, subfolder)
494
494
495 if os.path.exists(fullpath):
495 if os.path.exists(fullpath):
496 filesList = os.listdir(fullpath)
496 filesList = os.listdir(fullpath)
497 filesList = [k for k in filesList if k.startswith(self.optchar)]
497 filesList = [k for k in filesList if k.startswith(self.optchar)]
498 if len( filesList ) > 0:
498 if len( filesList ) > 0:
499 filesList = sorted(filesList, key=str.lower)
499 filesList = sorted(filesList, key=str.lower)
500 filen = filesList[-1]
500 filen = filesList[-1]
501 # el filename debera tener el siguiente formato
501 # el filename debera tener el siguiente formato
502 # 0 1234 567 89A BCDE (hex)
502 # 0 1234 567 89A BCDE (hex)
503 # x YYYY DDD SSS .ext
503 # x YYYY DDD SSS .ext
504 if isNumber(filen[8:11]):
504 if isNumber(filen[8:11]):
505 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
505 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
506 else:
506 else:
507 setFile = -1
507 setFile = -1
508 else:
508 else:
509 setFile = -1 #inicializo mi contador de seteo
509 setFile = -1 #inicializo mi contador de seteo
510 else:
510 else:
511 os.makedirs(fullpath)
511 os.makedirs(fullpath)
512 setFile = -1 #inicializo mi contador de seteo
512 setFile = -1 #inicializo mi contador de seteo
513
513
514 if self.setType is None:
514 if self.setType is None:
515 setFile += 1
515 setFile += 1
516 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
516 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
517 timeTuple.tm_year,
517 timeTuple.tm_year,
518 timeTuple.tm_yday,
518 timeTuple.tm_yday,
519 setFile,
519 setFile,
520 ext )
520 ext )
521 else:
521 else:
522 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
522 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
523 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
523 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
524 timeTuple.tm_year,
524 timeTuple.tm_year,
525 timeTuple.tm_yday,
525 timeTuple.tm_yday,
526 setFile,
526 setFile,
527 ext )
527 ext )
528
528
529 self.filename = os.path.join( path, subfolder, file )
529 self.filename = os.path.join( path, subfolder, file )
530
530
531 #Setting HDF5 File
531 #Setting HDF5 File
532 self.fp = h5py.File(self.filename, 'w')
532 self.fp = h5py.File(self.filename, 'w')
533 #write metadata
533 #write metadata
534 self.writeMetadata(self.fp)
534 self.writeMetadata(self.fp)
535 #Write data
535 #Write data
536 self.writeData(self.fp)
536 self.writeData(self.fp)
537
537
538 def getLabel(self, name, x=None):
538 def getLabel(self, name, x=None):
539
539
540 if x is None:
540 if x is None:
541 if 'Data' in self.description:
541 if 'Data' in self.description:
542 data = self.description['Data']
542 data = self.description['Data']
543 if 'Metadata' in self.description:
543 if 'Metadata' in self.description:
544 data.update(self.description['Metadata'])
544 data.update(self.description['Metadata'])
545 else:
545 else:
546 data = self.description
546 data = self.description
547 if name in data:
547 if name in data:
548 if isinstance(data[name], str):
548 if isinstance(data[name], str):
549 return data[name]
549 return data[name]
550 elif isinstance(data[name], list):
550 elif isinstance(data[name], list):
551 return None
551 return None
552 elif isinstance(data[name], dict):
552 elif isinstance(data[name], dict):
553 for key, value in data[name].items():
553 for key, value in data[name].items():
554 return key
554 return key
555 return name
555 return name
556 else:
556 else:
557 if 'Metadata' in self.description:
557 if 'Metadata' in self.description:
558 meta = self.description['Metadata']
558 meta = self.description['Metadata']
559 else:
559 else:
560 meta = self.description
560 meta = self.description
561 if name in meta:
561 if name in meta:
562 if isinstance(meta[name], list):
562 if isinstance(meta[name], list):
563 return meta[name][x]
563 return meta[name][x]
564 elif isinstance(meta[name], dict):
564 elif isinstance(meta[name], dict):
565 for key, value in meta[name].items():
565 for key, value in meta[name].items():
566 return value[x]
566 return value[x]
567 if 'cspc' in name:
567 if 'cspc' in name:
568 return 'pair{:02d}'.format(x)
568 return 'pair{:02d}'.format(x)
569 else:
569 else:
570 return 'channel{:02d}'.format(x)
570 return 'channel{:02d}'.format(x)
571
571
572 def writeMetadata(self, fp):
572 def writeMetadata(self, fp):
573
573
574 if self.description:
574 if self.description:
575 if 'Metadata' in self.description:
575 if 'Metadata' in self.description:
576 grp = fp.create_group('Metadata')
576 grp = fp.create_group('Metadata')
577 else:
577 else:
578 grp = fp
578 grp = fp
579 else:
579 else:
580 grp = fp.create_group('Metadata')
580 grp = fp.create_group('Metadata')
581
581
582 for i in range(len(self.metadataList)):
582 for i in range(len(self.metadataList)):
583 if not hasattr(self.dataOut, self.metadataList[i]):
583 if not hasattr(self.dataOut, self.metadataList[i]):
584 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
584 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
585 continue
585 continue
586 value = getattr(self.dataOut, self.metadataList[i])
586 value = getattr(self.dataOut, self.metadataList[i])
587 if isinstance(value, bool):
587 if isinstance(value, bool):
588 if value is True:
588 if value is True:
589 value = 1
589 value = 1
590 else:
590 else:
591 value = 0
591 value = 0
592 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
592 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
593 return
593 return
594
594
595 def writeData(self, fp):
595 def writeData(self, fp):
596
596
597 if self.description:
597 if self.description:
598 if 'Data' in self.description:
598 if 'Data' in self.description:
599 grp = fp.create_group('Data')
599 grp = fp.create_group('Data')
600 else:
600 else:
601 grp = fp
601 grp = fp
602 else:
602 else:
603 grp = fp.create_group('Data')
603 grp = fp.create_group('Data')
604
604
605 dtsets = []
605 dtsets = []
606 data = []
606 data = []
607
607
608 for dsInfo in self.dsList:
608 for dsInfo in self.dsList:
609 if dsInfo['nDim'] == 0:
609 if dsInfo['nDim'] == 0:
610 ds = grp.create_dataset(
610 ds = grp.create_dataset(
611 self.getLabel(dsInfo['variable']),
611 self.getLabel(dsInfo['variable']),
612 (self.blocksPerFile, ),
612 (self.blocksPerFile, ),
613 chunks=True,
613 chunks=True,
614 dtype=numpy.float64)
614 dtype=numpy.float64)
615 dtsets.append(ds)
615 dtsets.append(ds)
616 data.append((dsInfo['variable'], -1))
616 data.append((dsInfo['variable'], -1))
617 else:
617 else:
618 label = self.getLabel(dsInfo['variable'])
618 label = self.getLabel(dsInfo['variable'])
619 if label is not None:
619 if label is not None:
620 sgrp = grp.create_group(label)
620 sgrp = grp.create_group(label)
621 else:
621 else:
622 sgrp = grp
622 sgrp = grp
623 for i in range(dsInfo['dsNumber']):
623 for i in range(dsInfo['dsNumber']):
624 ds = sgrp.create_dataset(
624 ds = sgrp.create_dataset(
625 self.getLabel(dsInfo['variable'], i),
625 self.getLabel(dsInfo['variable'], i),
626 (self.blocksPerFile, ) + dsInfo['shape'][1:],
626 (self.blocksPerFile, ) + dsInfo['shape'][1:],
627 chunks=True,
627 chunks=True,
628 dtype=dsInfo['dtype'])
628 dtype=dsInfo['dtype'])
629 dtsets.append(ds)
629 dtsets.append(ds)
630 data.append((dsInfo['variable'], i))
630 data.append((dsInfo['variable'], i))
631 fp.flush()
631 fp.flush()
632
632
633 log.log('Creating file: {}'.format(fp.filename), self.name)
633 log.log('Creating file: {}'.format(fp.filename), self.name)
634
634
635 self.ds = dtsets
635 self.ds = dtsets
636 self.data = data
636 self.data = data
637 self.firsttime = True
637 self.firsttime = True
638 self.blockIndex = 0
638 self.blockIndex = 0
639 return
639 return
640
640
641 def putData(self):
641 def putData(self):
642
642
643 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
643 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
644 self.closeFile()
644 self.closeFile()
645 self.setNextFile()
645 self.setNextFile()
646
646
647 for i, ds in enumerate(self.ds):
647 for i, ds in enumerate(self.ds):
648 attr, ch = self.data[i]
648 attr, ch = self.data[i]
649 if ch == -1:
649 if ch == -1:
650 ds[self.blockIndex] = getattr(self.dataOut, attr)
650 ds[self.blockIndex] = getattr(self.dataOut, attr)
651 else:
651 else:
652 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
652 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
653
653
654 self.fp.flush()
654 self.fp.flush()
655 self.blockIndex += 1
655 self.blockIndex += 1
656 if self.blockIndex == 1:
656 if self.blockIndex == 1:
657 log.log('Block No. {}/{} --> {}'.format(self.blockIndex, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
657 log.log('Block No. {}/{} --> {}'.format(self.blockIndex, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
658 else:
658 else:
659 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
659 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
660 return
660 return
661
661
662 def closeFile(self):
662 def closeFile(self):
663
663
664 if self.blockIndex != self.blocksPerFile:
664 if self.blockIndex != self.blocksPerFile:
665 for ds in self.ds:
665 for ds in self.ds:
666 ds.resize(self.blockIndex, axis=0)
666 ds.resize(self.blockIndex, axis=0)
667
667
668 if self.fp:
668 if self.fp:
669 self.fp.flush()
669 self.fp.flush()
670 self.fp.close()
670 self.fp.close()
671
671
672 def close(self):
672 def close(self):
673
673
674 self.closeFile()
674 self.closeFile()
General Comments 0
You need to be logged in to leave comments. Login now