##// END OF EJS Templates
schain
RSS
Límite de bloques, hora, zona horaria
joabAM -
r1401:c018ae3094fa
parent child
Show More
Side by Side Unified
Context file:
r1401:c018ae3094fa Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,660 +1,674
1 import os
1 import os
2 import time
2 import time
3 import datetime
3 import datetime
4
4
5 import numpy
5 import numpy
6 import h5py
6 import h5py
7
7
8 import schainpy.admin
8 import schainpy.admin
9 from schainpy.model.data.jrodata import *
9 from schainpy.model.data.jrodata import *
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 from schainpy.utils import log
12 from schainpy.utils import log
13
13
14
14
15 class HDFReader(Reader, ProcessingUnit):
15 class HDFReader(Reader, ProcessingUnit):
16 """Processing unit to read HDF5 format files
16 """Processing unit to read HDF5 format files
17
17
18 This unit reads HDF5 files created with `HDFWriter` operation contains
18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 attributes.
20 attributes.
21 It is possible to read any HDF5 file by given the structure in the `description`
21 It is possible to read any HDF5 file by given the structure in the `description`
22 parameter, also you can add extra values to metadata with the parameter `extras`.
22 parameter, also you can add extra values to metadata with the parameter `extras`.
23
23
24 Parameters:
24 Parameters:
25 -----------
25 -----------
26 path : str
26 path : str
27 Path where files are located.
27 Path where files are located.
28 startDate : date
28 startDate : date
29 Start date of the files
29 Start date of the files
30 endDate : list
30 endDate : list
31 End date of the files
31 End date of the files
32 startTime : time
32 startTime : time
33 Start time of the files
33 Start time of the files
34 endTime : time
34 endTime : time
35 End time of the files
35 End time of the files
36 description : dict, optional
36 description : dict, optional
37 Dictionary with the description of the HDF5 file
37 Dictionary with the description of the HDF5 file
38 extras : dict, optional
38 extras : dict, optional
39 Dictionary with extra metadata to be be added to `dataOut`
39 Dictionary with extra metadata to be be added to `dataOut`
40
40
41 Examples
41 Examples
42 --------
42 --------
43
43
44 desc = {
44 desc = {
45 'Data': {
45 'Data': {
46 'data_output': ['u', 'v', 'w'],
46 'data_output': ['u', 'v', 'w'],
47 'utctime': 'timestamps',
47 'utctime': 'timestamps',
48 } ,
48 } ,
49 'Metadata': {
49 'Metadata': {
50 'heightList': 'heights'
50 'heightList': 'heights'
51 }
51 }
52 }
52 }
53
53
54 desc = {
54 desc = {
55 'Data': {
55 'Data': {
56 'data_output': 'winds',
56 'data_output': 'winds',
57 'utctime': 'timestamps'
57 'utctime': 'timestamps'
58 },
58 },
59 'Metadata': {
59 'Metadata': {
60 'heightList': 'heights'
60 'heightList': 'heights'
61 }
61 }
62 }
62 }
63
63
64 extras = {
64 extras = {
65 'timeZone': 300
65 'timeZone': 300
66 }
66 }
67
67
68 reader = project.addReadUnit(
68 reader = project.addReadUnit(
69 name='HDFReader',
69 name='HDFReader',
70 path='/path/to/files',
70 path='/path/to/files',
71 startDate='2019/01/01',
71 startDate='2019/01/01',
72 endDate='2019/01/31',
72 endDate='2019/01/31',
73 startTime='00:00:00',
73 startTime='00:00:00',
74 endTime='23:59:59',
74 endTime='23:59:59',
75 # description=json.dumps(desc),
75 # description=json.dumps(desc),
76 # extras=json.dumps(extras),
76 # extras=json.dumps(extras),
77 )
77 )
78
78
79 """
79 """
80
80
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82
82
83 def __init__(self):
83 def __init__(self):
84 ProcessingUnit.__init__(self)
84 ProcessingUnit.__init__(self)
85 self.dataOut = Parameters()
85 self.dataOut = Parameters()
86 self.ext = ".hdf5"
86 self.ext = ".hdf5"
87 self.optchar = "D"
87 self.optchar = "D"
88 self.meta = {}
88 self.meta = {}
89 self.data = {}
89 self.data = {}
90 self.open_file = h5py.File
90 self.open_file = h5py.File
91 self.open_mode = 'r'
91 self.open_mode = 'r'
92 self.description = {}
92 self.description = {}
93 self.extras = {}
93 self.extras = {}
94 self.filefmt = "*%Y%j***"
94 self.filefmt = "*%Y%j***"
95 self.folderfmt = "*%Y%j"
95 self.folderfmt = "*%Y%j"
96 self.utcoffset = 0
96 self.utcoffset = 0
97
97
98 def setup(self, **kwargs):
98 def setup(self, **kwargs):
99
99
100 self.set_kwargs(**kwargs)
100 self.set_kwargs(**kwargs)
101 if not self.ext.startswith('.'):
101 if not self.ext.startswith('.'):
102 self.ext = '.{}'.format(self.ext)
102 self.ext = '.{}'.format(self.ext)
103
103
104 if self.online:
104 if self.online:
105 log.log("Searching files in online mode...", self.name)
105 log.log("Searching files in online mode...", self.name)
106
106
107 for nTries in range(self.nTries):
107 for nTries in range(self.nTries):
108 fullpath = self.searchFilesOnLine(self.path, self.startDate,
108 fullpath = self.searchFilesOnLine(self.path, self.startDate,
109 self.endDate, self.expLabel, self.ext, self.walk,
109 self.endDate, self.expLabel, self.ext, self.walk,
110 self.filefmt, self.folderfmt)
110 self.filefmt, self.folderfmt)
111 pathname, filename = os.path.split(fullpath)
111 pathname, filename = os.path.split(fullpath)
112 #print(pathname,filename)
112 #print(pathname,filename)
113 try:
113 try:
114 fullpath = next(fullpath)
114 fullpath = next(fullpath)
115
115
116 except:
116 except:
117 fullpath = None
117 fullpath = None
118
118
119 if fullpath:
119 if fullpath:
120 break
120 break
121
121
122 log.warning(
122 log.warning(
123 'Waiting {} sec for a valid file in {}: try {} ...'.format(
123 'Waiting {} sec for a valid file in {}: try {} ...'.format(
124 self.delay, self.path, nTries + 1),
124 self.delay, self.path, nTries + 1),
125 self.name)
125 self.name)
126 time.sleep(self.delay)
126 time.sleep(self.delay)
127
127
128 if not(fullpath):
128 if not(fullpath):
129 raise schainpy.admin.SchainError(
129 raise schainpy.admin.SchainError(
130 'There isn\'t any valid file in {}'.format(self.path))
130 'There isn\'t any valid file in {}'.format(self.path))
131
131
132 pathname, filename = os.path.split(fullpath)
132 pathname, filename = os.path.split(fullpath)
133 self.year = int(filename[1:5])
133 self.year = int(filename[1:5])
134 self.doy = int(filename[5:8])
134 self.doy = int(filename[5:8])
135 self.set = int(filename[8:11]) - 1
135 self.set = int(filename[8:11]) - 1
136 else:
136 else:
137 log.log("Searching files in {}".format(self.path), self.name)
137 log.log("Searching files in {}".format(self.path), self.name)
138 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
138 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
139 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
139 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
140
140
141 self.setNextFile()
141 self.setNextFile()
142
142
143 return
143 return
144
144
145
145
146 def readFirstHeader(self):
146 def readFirstHeader(self):
147 '''Read metadata and data'''
147 '''Read metadata and data'''
148
148
149 self.__readMetadata()
149 self.__readMetadata()
150 self.__readData()
150 self.__readData()
151 self.__setBlockList()
151 self.__setBlockList()
152
152
153 if 'type' in self.meta:
153 if 'type' in self.meta:
154 ##print("Creting dataOut...")
154 self.dataOut = eval(self.meta['type'])()
155 self.dataOut = eval(self.meta['type'])()
156 ##print(vars(self.dataOut))
155
157
156 for attr in self.meta:
158 for attr in self.meta:
157 #print("attr: ", attr)
159 ##print("attr: ", attr)
160 ##print(type(self.dataOut).__name__)
158 setattr(self.dataOut, attr, self.meta[attr])
161 setattr(self.dataOut, attr, self.meta[attr])
159
162
160
161 self.blockIndex = 0
163 self.blockIndex = 0
162
164
163 return
165 return
164
166
165 def __setBlockList(self):
167 def __setBlockList(self):
166 '''
168 '''
167 Selects the data within the times defined
169 Selects the data within the times defined
168
170
169 self.fp
171 self.fp
170 self.startTime
172 self.startTime
171 self.endTime
173 self.endTime
172 self.blockList
174 self.blockList
173 self.blocksPerFile
175 self.blocksPerFile
174
176
175 '''
177 '''
176
178
177 startTime = self.startTime
179 startTime = self.startTime
178 endTime = self.endTime
180 endTime = self.endTime
179 thisUtcTime = self.data['utctime'] + self.utcoffset
181 thisUtcTime = self.data['utctime'] + self.utcoffset
180 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
182 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
181 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
183 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
182 self.startFileDatetime = thisDatetime
184 self.startFileDatetime = thisDatetime
183 thisDate = thisDatetime.date()
185 thisDate = thisDatetime.date()
184 thisTime = thisDatetime.time()
186 thisTime = thisDatetime.time()
185
187
186 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
188 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
187 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
189 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
188
190
189 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
191 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
190
192
191 self.blockList = ind
193 self.blockList = ind
192 self.blocksPerFile = len(ind)
194 self.blocksPerFile = len(ind)
193 self.blocksPerFile = len(thisUtcTime)
195 self.blocksPerFile = len(thisUtcTime)
194 return
196 return
195
197
196 def __readMetadata(self):
198 def __readMetadata(self):
197 '''
199 '''
198 Reads Metadata
200 Reads Metadata
199 '''
201 '''
200
202
201 meta = {}
203 meta = {}
202
204
203 if self.description:
205 if self.description:
204 for key, value in self.description['Metadata'].items():
206 for key, value in self.description['Metadata'].items():
205 meta[key] = self.fp[value][()]
207 meta[key] = self.fp[value][()]
206 else:
208 else:
207 grp = self.fp['Metadata']
209 grp = self.fp['Metadata']
208 for name in grp:
210 for name in grp:
209 meta[name] = grp[name][()]
211 meta[name] = grp[name][()]
210
212
211 if self.extras:
213 if self.extras:
212 for key, value in self.extras.items():
214 for key, value in self.extras.items():
213 meta[key] = value
215 meta[key] = value
214 self.meta = meta
216 self.meta = meta
215
217
216 return
218 return
217
219
218
220
219
221
220 def checkForRealPath(self, nextFile, nextDay):
222 def checkForRealPath(self, nextFile, nextDay):
221
223
222 # print("check FRP")
224 # print("check FRP")
223 # dt = self.startFileDatetime + datetime.timedelta(1)
225 # dt = self.startFileDatetime + datetime.timedelta(1)
224 # filename = '{}.{}{}'.format(self.path, dt.strftime('%Y%m%d'), self.ext)
226 # filename = '{}.{}{}'.format(self.path, dt.strftime('%Y%m%d'), self.ext)
225 # fullfilename = os.path.join(self.path, filename)
227 # fullfilename = os.path.join(self.path, filename)
226 # print("check Path ",fullfilename,filename)
228 # print("check Path ",fullfilename,filename)
227 # if os.path.exists(fullfilename):
229 # if os.path.exists(fullfilename):
228 # return fullfilename, filename
230 # return fullfilename, filename
229 # return None, filename
231 # return None, filename
230 return None,None
232 return None,None
231
233
232 def __readData(self):
234 def __readData(self):
233
235
234 data = {}
236 data = {}
235
237
236 if self.description:
238 if self.description:
237 for key, value in self.description['Data'].items():
239 for key, value in self.description['Data'].items():
238 if isinstance(value, str):
240 if isinstance(value, str):
239 if isinstance(self.fp[value], h5py.Dataset):
241 if isinstance(self.fp[value], h5py.Dataset):
240 data[key] = self.fp[value][()]
242 data[key] = self.fp[value][()]
241 elif isinstance(self.fp[value], h5py.Group):
243 elif isinstance(self.fp[value], h5py.Group):
242 array = []
244 array = []
243 for ch in self.fp[value]:
245 for ch in self.fp[value]:
244 array.append(self.fp[value][ch][()])
246 array.append(self.fp[value][ch][()])
245 data[key] = numpy.array(array)
247 data[key] = numpy.array(array)
246 elif isinstance(value, list):
248 elif isinstance(value, list):
247 array = []
249 array = []
248 for ch in value:
250 for ch in value:
249 array.append(self.fp[ch][()])
251 array.append(self.fp[ch][()])
250 data[key] = numpy.array(array)
252 data[key] = numpy.array(array)
251 else:
253 else:
252 grp = self.fp['Data']
254 grp = self.fp['Data']
253 for name in grp:
255 for name in grp:
254 if isinstance(grp[name], h5py.Dataset):
256 if isinstance(grp[name], h5py.Dataset):
255 array = grp[name][()]
257 array = grp[name][()]
256 elif isinstance(grp[name], h5py.Group):
258 elif isinstance(grp[name], h5py.Group):
257 array = []
259 array = []
258 for ch in grp[name]:
260 for ch in grp[name]:
259 array.append(grp[name][ch][()])
261 array.append(grp[name][ch][()])
260 array = numpy.array(array)
262 array = numpy.array(array)
261 else:
263 else:
262 log.warning('Unknown type: {}'.format(name))
264 log.warning('Unknown type: {}'.format(name))
263
265
264 if name in self.description:
266 if name in self.description:
265 key = self.description[name]
267 key = self.description[name]
266 else:
268 else:
267 key = name
269 key = name
268 data[key] = array
270 data[key] = array
269
271
270 self.data = data
272 self.data = data
271 return
273 return
272
274
273 def getData(self):
275 def getData(self):
274 if not self.isDateTimeInRange(self.startFileDatetime, self.startDate, self.endDate, self.startTime, self.endTime):
276 if not self.isDateTimeInRange(self.startFileDatetime, self.startDate, self.endDate, self.startTime, self.endTime):
275 self.dataOut.flagNoData = True
277 self.dataOut.flagNoData = True
276 self.blockIndex = self.blocksPerFile
278 self.blockIndex = self.blocksPerFile
277 #self.dataOut.error = True TERMINA EL PROGRAMA, removido
279 #self.dataOut.error = True TERMINA EL PROGRAMA, removido
278 return
280 return
279 for attr in self.data:
281 for attr in self.data:
282 #print("attr ",attr)
280 if self.data[attr].ndim == 1:
283 if self.data[attr].ndim == 1:
281 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
284 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
282 else:
285 else:
283 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
286 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
284
287
285 self.dataOut.flagNoData = False
288
286 self.blockIndex += 1
289 self.blockIndex += 1
287
290
288 if self.blockIndex == 1:
291 if self.blockIndex == 1:
289 log.log("Block No. {}/{} -> {}".format(
292 log.log("Block No. {}/{} -> {}".format(
290 self.blockIndex,
293 self.blockIndex,
291 self.blocksPerFile,
294 self.blocksPerFile,
292 self.dataOut.datatime.ctime()), self.name)
295 self.dataOut.datatime.ctime()), self.name)
293 else:
296 else:
294 log.log("Block No. {}/{} ".format(
297 log.log("Block No. {}/{} ".format(
295 self.blockIndex,
298 self.blockIndex,
296 self.blocksPerFile),self.name)
299 self.blocksPerFile),self.name)
297
300
298
301 self.dataOut.flagNoData = False
302 self.dataOut.error = False
299 return
303 return
300
304
301 def run(self, **kwargs):
305 def run(self, **kwargs):
302
306
303 if not(self.isConfig):
307 if not(self.isConfig):
304 self.setup(**kwargs)
308 self.setup(**kwargs)
305 self.isConfig = True
309 self.isConfig = True
306
310
307 if self.blockIndex == self.blocksPerFile:
311 if self.blockIndex == self.blocksPerFile:
308 self.setNextFile()
312 self.setNextFile()
309
313
310 self.getData()
314 self.getData()
311
315
312 return
316 return
313
317
314 @MPDecorator
318 @MPDecorator
315 class HDFWriter(Operation):
319 class HDFWriter(Operation):
316 """Operation to write HDF5 files.
320 """Operation to write HDF5 files.
317
321
318 The HDF5 file contains by default two groups Data and Metadata where
322 The HDF5 file contains by default two groups Data and Metadata where
319 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
323 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
320 parameters, data attributes are normaly time dependent where the metadata
324 parameters, data attributes are normaly time dependent where the metadata
321 are not.
325 are not.
322 It is possible to customize the structure of the HDF5 file with the
326 It is possible to customize the structure of the HDF5 file with the
323 optional description parameter see the examples.
327 optional description parameter see the examples.
324
328
325 Parameters:
329 Parameters:
326 -----------
330 -----------
327 path : str
331 path : str
328 Path where files will be saved.
332 Path where files will be saved.
329 blocksPerFile : int
333 blocksPerFile : int
330 Number of blocks per file
334 Number of blocks per file
331 metadataList : list
335 metadataList : list
332 List of the dataOut attributes that will be saved as metadata
336 List of the dataOut attributes that will be saved as metadata
333 dataList : int
337 dataList : int
334 List of the dataOut attributes that will be saved as data
338 List of the dataOut attributes that will be saved as data
335 setType : bool
339 setType : bool
336 If True the name of the files corresponds to the timestamp of the data
340 If True the name of the files corresponds to the timestamp of the data
337 description : dict, optional
341 description : dict, optional
338 Dictionary with the desired description of the HDF5 file
342 Dictionary with the desired description of the HDF5 file
339
343
340 Examples
344 Examples
341 --------
345 --------
342
346
343 desc = {
347 desc = {
344 'data_output': {'winds': ['z', 'w', 'v']},
348 'data_output': {'winds': ['z', 'w', 'v']},
345 'utctime': 'timestamps',
349 'utctime': 'timestamps',
346 'heightList': 'heights'
350 'heightList': 'heights'
347 }
351 }
348 desc = {
352 desc = {
349 'data_output': ['z', 'w', 'v'],
353 'data_output': ['z', 'w', 'v'],
350 'utctime': 'timestamps',
354 'utctime': 'timestamps',
351 'heightList': 'heights'
355 'heightList': 'heights'
352 }
356 }
353 desc = {
357 desc = {
354 'Data': {
358 'Data': {
355 'data_output': 'winds',
359 'data_output': 'winds',
356 'utctime': 'timestamps'
360 'utctime': 'timestamps'
357 },
361 },
358 'Metadata': {
362 'Metadata': {
359 'heightList': 'heights'
363 'heightList': 'heights'
360 }
364 }
361 }
365 }
362
366
363 writer = proc_unit.addOperation(name='HDFWriter')
367 writer = proc_unit.addOperation(name='HDFWriter')
364 writer.addParameter(name='path', value='/path/to/file')
368 writer.addParameter(name='path', value='/path/to/file')
365 writer.addParameter(name='blocksPerFile', value='32')
369 writer.addParameter(name='blocksPerFile', value='32')
366 writer.addParameter(name='metadataList', value='heightList,timeZone')
370 writer.addParameter(name='metadataList', value='heightList,timeZone')
367 writer.addParameter(name='dataList',value='data_output,utctime')
371 writer.addParameter(name='dataList',value='data_output,utctime')
368 # writer.addParameter(name='description',value=json.dumps(desc))
372 # writer.addParameter(name='description',value=json.dumps(desc))
369
373
370 """
374 """
371
375
372 ext = ".hdf5"
376 ext = ".hdf5"
373 optchar = "D"
377 optchar = "D"
374 filename = None
378 filename = None
375 path = None
379 path = None
376 setFile = None
380 setFile = None
377 fp = None
381 fp = None
378 firsttime = True
382 firsttime = True
379 #Configurations
383 #Configurations
380 blocksPerFile = None
384 blocksPerFile = None
381 blockIndex = None
385 blockIndex = None
382 dataOut = None
386 dataOut = None
383 #Data Arrays
387 #Data Arrays
384 dataList = None
388 dataList = None
385 metadataList = None
389 metadataList = None
386 currentDay = None
390 currentDay = None
387 lastTime = None
391 lastTime = None
392 typeTime = "ut"
393 hourLimit = 3
394 breakDays = True
388
395
389 def __init__(self):
396 def __init__(self):
390
397
391 Operation.__init__(self)
398 Operation.__init__(self)
392 return
399 return
393
400
394 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
401 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None,
402 description=None,typeTime = "ut",hourLimit = 3, breakDays=True):
395 self.path = path
403 self.path = path
396 self.blocksPerFile = blocksPerFile
404 self.blocksPerFile = blocksPerFile
397 self.metadataList = metadataList
405 self.metadataList = metadataList
398 self.dataList = [s.strip() for s in dataList]
406 self.dataList = [s.strip() for s in dataList]
399 self.setType = setType
407 self.setType = setType
400 self.description = description
408 self.description = description
409 self.timeZone = typeTime
410 self.hourLimit = hourLimit
411 self.breakDays = breakDays
401
412
402 if self.metadataList is None:
413 if self.metadataList is None:
403 self.metadataList = self.dataOut.metadata_list
414 self.metadataList = self.dataOut.metadata_list
404
415
405 tableList = []
416 tableList = []
406 dsList = []
417 dsList = []
407
418
408 for i in range(len(self.dataList)):
419 for i in range(len(self.dataList)):
409 dsDict = {}
420 dsDict = {}
410 if hasattr(self.dataOut, self.dataList[i]):
421 if hasattr(self.dataOut, self.dataList[i]):
411 dataAux = getattr(self.dataOut, self.dataList[i])
422 dataAux = getattr(self.dataOut, self.dataList[i])
412 dsDict['variable'] = self.dataList[i]
423 dsDict['variable'] = self.dataList[i]
413 else:
424 else:
414 log.warning('Attribute {} not found in dataOut', self.name)
425 log.warning('Attribute {} not found in dataOut', self.name)
415 continue
426 continue
416
427
417 if dataAux is None:
428 if dataAux is None:
418 continue
429 continue
419 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
430 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
420 dsDict['nDim'] = 0
431 dsDict['nDim'] = 0
421 else:
432 else:
422 dsDict['nDim'] = len(dataAux.shape)
433 dsDict['nDim'] = len(dataAux.shape)
423 dsDict['shape'] = dataAux.shape
434 dsDict['shape'] = dataAux.shape
424 dsDict['dsNumber'] = dataAux.shape[0]
435 dsDict['dsNumber'] = dataAux.shape[0]
425 dsDict['dtype'] = dataAux.dtype
436 dsDict['dtype'] = dataAux.dtype
426
437
427 dsList.append(dsDict)
438 dsList.append(dsDict)
428
439
429 self.dsList = dsList
440 self.dsList = dsList
430 self.currentDay = self.dataOut.datatime.date()
441 self.currentDay = self.dataOut.datatime.date()
431
442
432 def timeFlag(self):
443 def timeFlag(self):
433 currentTime = self.dataOut.utctime
444 currentTime = self.dataOut.utctime
445 if self.timeZone == "lt":
434 timeTuple = time.localtime(currentTime)
446 timeTuple = time.localtime(currentTime)
447 elif self.timeZone == "ut":
448 timeTuple = time.gmtime(currentTime)
449
435 dataDay = timeTuple.tm_yday
450 dataDay = timeTuple.tm_yday
436 #print("time UTC: ",currentTime, self.dataOut.datatime)
451 #print("time UTC: ",currentTime, self.dataOut.datatime)
437 if self.lastTime is None:
452 if self.lastTime is None:
438 self.lastTime = currentTime
453 self.lastTime = currentTime
439 self.currentDay = dataDay
454 self.currentDay = dataDay
440 return False
455 return False
441
456
442 timeDiff = currentTime - self.lastTime
457 timeDiff = currentTime - self.lastTime
443
458
444 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
459 #Si el dia es diferente o si la diferencia entre un dato y otro supera self.hourLimit
445 if dataDay != self.currentDay:
460 if (dataDay != self.currentDay) and self.breakDays:
446 self.currentDay = dataDay
461 self.currentDay = dataDay
447 return True
462 return True
448 elif timeDiff > 3*60*60:
463 elif timeDiff > self.hourLimit*60*60:
449 self.lastTime = currentTime
464 self.lastTime = currentTime
450 return True
465 return True
451 else:
466 else:
452 self.lastTime = currentTime
467 self.lastTime = currentTime
453 return False
468 return False
454
469
455 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
470 def run(self, dataOut,**kwargs):
456 dataList=[], setType=None, description={}):
457
471
458 self.dataOut = dataOut
472 self.dataOut = dataOut
459 if not(self.isConfig):
473 if not(self.isConfig):
460 self.setup(path=path, blocksPerFile=blocksPerFile,
474 self.setup(**kwargs)
461 metadataList=metadataList, dataList=dataList,
462 setType=setType, description=description)
463
475
464 self.isConfig = True
476 self.isConfig = True
465 self.setNextFile()
477 self.setNextFile()
466
478
467 self.putData()
479 self.putData()
468 return
480 return
469
481
470 def setNextFile(self):
482 def setNextFile(self):
471
483
472 ext = self.ext
484 ext = self.ext
473 path = self.path
485 path = self.path
474 setFile = self.setFile
486 setFile = self.setFile
475
487 if self.timeZone == "lt":
488 timeTuple = time.localtime(self.dataOut.utctime)
489 elif self.timeZone == "ut":
476 timeTuple = time.gmtime(self.dataOut.utctime)
490 timeTuple = time.gmtime(self.dataOut.utctime)
477 #print("path: ",timeTuple)
491 #print("path: ",timeTuple)
478 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
492 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
479 fullpath = os.path.join(path, subfolder)
493 fullpath = os.path.join(path, subfolder)
480
494
481 if os.path.exists(fullpath):
495 if os.path.exists(fullpath):
482 filesList = os.listdir(fullpath)
496 filesList = os.listdir(fullpath)
483 filesList = [k for k in filesList if k.startswith(self.optchar)]
497 filesList = [k for k in filesList if k.startswith(self.optchar)]
484 if len( filesList ) > 0:
498 if len( filesList ) > 0:
485 filesList = sorted(filesList, key=str.lower)
499 filesList = sorted(filesList, key=str.lower)
486 filen = filesList[-1]
500 filen = filesList[-1]
487 # el filename debera tener el siguiente formato
501 # el filename debera tener el siguiente formato
488 # 0 1234 567 89A BCDE (hex)
502 # 0 1234 567 89A BCDE (hex)
489 # x YYYY DDD SSS .ext
503 # x YYYY DDD SSS .ext
490 if isNumber(filen[8:11]):
504 if isNumber(filen[8:11]):
491 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
505 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
492 else:
506 else:
493 setFile = -1
507 setFile = -1
494 else:
508 else:
495 setFile = -1 #inicializo mi contador de seteo
509 setFile = -1 #inicializo mi contador de seteo
496 else:
510 else:
497 os.makedirs(fullpath)
511 os.makedirs(fullpath)
498 setFile = -1 #inicializo mi contador de seteo
512 setFile = -1 #inicializo mi contador de seteo
499
513
500 if self.setType is None:
514 if self.setType is None:
501 setFile += 1
515 setFile += 1
502 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
516 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
503 timeTuple.tm_year,
517 timeTuple.tm_year,
504 timeTuple.tm_yday,
518 timeTuple.tm_yday,
505 setFile,
519 setFile,
506 ext )
520 ext )
507 else:
521 else:
508 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
522 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
509 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
523 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
510 timeTuple.tm_year,
524 timeTuple.tm_year,
511 timeTuple.tm_yday,
525 timeTuple.tm_yday,
512 setFile,
526 setFile,
513 ext )
527 ext )
514
528
515 self.filename = os.path.join( path, subfolder, file )
529 self.filename = os.path.join( path, subfolder, file )
516
530
517 #Setting HDF5 File
531 #Setting HDF5 File
518 self.fp = h5py.File(self.filename, 'w')
532 self.fp = h5py.File(self.filename, 'w')
519 #write metadata
533 #write metadata
520 self.writeMetadata(self.fp)
534 self.writeMetadata(self.fp)
521 #Write data
535 #Write data
522 self.writeData(self.fp)
536 self.writeData(self.fp)
523
537
524 def getLabel(self, name, x=None):
538 def getLabel(self, name, x=None):
525
539
526 if x is None:
540 if x is None:
527 if 'Data' in self.description:
541 if 'Data' in self.description:
528 data = self.description['Data']
542 data = self.description['Data']
529 if 'Metadata' in self.description:
543 if 'Metadata' in self.description:
530 data.update(self.description['Metadata'])
544 data.update(self.description['Metadata'])
531 else:
545 else:
532 data = self.description
546 data = self.description
533 if name in data:
547 if name in data:
534 if isinstance(data[name], str):
548 if isinstance(data[name], str):
535 return data[name]
549 return data[name]
536 elif isinstance(data[name], list):
550 elif isinstance(data[name], list):
537 return None
551 return None
538 elif isinstance(data[name], dict):
552 elif isinstance(data[name], dict):
539 for key, value in data[name].items():
553 for key, value in data[name].items():
540 return key
554 return key
541 return name
555 return name
542 else:
556 else:
543 if 'Metadata' in self.description:
557 if 'Metadata' in self.description:
544 meta = self.description['Metadata']
558 meta = self.description['Metadata']
545 else:
559 else:
546 meta = self.description
560 meta = self.description
547 if name in meta:
561 if name in meta:
548 if isinstance(meta[name], list):
562 if isinstance(meta[name], list):
549 return meta[name][x]
563 return meta[name][x]
550 elif isinstance(meta[name], dict):
564 elif isinstance(meta[name], dict):
551 for key, value in meta[name].items():
565 for key, value in meta[name].items():
552 return value[x]
566 return value[x]
553 if 'cspc' in name:
567 if 'cspc' in name:
554 return 'pair{:02d}'.format(x)
568 return 'pair{:02d}'.format(x)
555 else:
569 else:
556 return 'channel{:02d}'.format(x)
570 return 'channel{:02d}'.format(x)
557
571
558 def writeMetadata(self, fp):
572 def writeMetadata(self, fp):
559
573
560 if self.description:
574 if self.description:
561 if 'Metadata' in self.description:
575 if 'Metadata' in self.description:
562 grp = fp.create_group('Metadata')
576 grp = fp.create_group('Metadata')
563 else:
577 else:
564 grp = fp
578 grp = fp
565 else:
579 else:
566 grp = fp.create_group('Metadata')
580 grp = fp.create_group('Metadata')
567
581
568 for i in range(len(self.metadataList)):
582 for i in range(len(self.metadataList)):
569 if not hasattr(self.dataOut, self.metadataList[i]):
583 if not hasattr(self.dataOut, self.metadataList[i]):
570 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
584 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
571 continue
585 continue
572 value = getattr(self.dataOut, self.metadataList[i])
586 value = getattr(self.dataOut, self.metadataList[i])
573 if isinstance(value, bool):
587 if isinstance(value, bool):
574 if value is True:
588 if value is True:
575 value = 1
589 value = 1
576 else:
590 else:
577 value = 0
591 value = 0
578 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
592 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
579 return
593 return
580
594
581 def writeData(self, fp):
595 def writeData(self, fp):
582
596
583 if self.description:
597 if self.description:
584 if 'Data' in self.description:
598 if 'Data' in self.description:
585 grp = fp.create_group('Data')
599 grp = fp.create_group('Data')
586 else:
600 else:
587 grp = fp
601 grp = fp
588 else:
602 else:
589 grp = fp.create_group('Data')
603 grp = fp.create_group('Data')
590
604
591 dtsets = []
605 dtsets = []
592 data = []
606 data = []
593
607
594 for dsInfo in self.dsList:
608 for dsInfo in self.dsList:
595 if dsInfo['nDim'] == 0:
609 if dsInfo['nDim'] == 0:
596 ds = grp.create_dataset(
610 ds = grp.create_dataset(
597 self.getLabel(dsInfo['variable']),
611 self.getLabel(dsInfo['variable']),
598 (self.blocksPerFile, ),
612 (self.blocksPerFile, ),
599 chunks=True,
613 chunks=True,
600 dtype=numpy.float64)
614 dtype=numpy.float64)
601 dtsets.append(ds)
615 dtsets.append(ds)
602 data.append((dsInfo['variable'], -1))
616 data.append((dsInfo['variable'], -1))
603 else:
617 else:
604 label = self.getLabel(dsInfo['variable'])
618 label = self.getLabel(dsInfo['variable'])
605 if label is not None:
619 if label is not None:
606 sgrp = grp.create_group(label)
620 sgrp = grp.create_group(label)
607 else:
621 else:
608 sgrp = grp
622 sgrp = grp
609 for i in range(dsInfo['dsNumber']):
623 for i in range(dsInfo['dsNumber']):
610 ds = sgrp.create_dataset(
624 ds = sgrp.create_dataset(
611 self.getLabel(dsInfo['variable'], i),
625 self.getLabel(dsInfo['variable'], i),
612 (self.blocksPerFile, ) + dsInfo['shape'][1:],
626 (self.blocksPerFile, ) + dsInfo['shape'][1:],
613 chunks=True,
627 chunks=True,
614 dtype=dsInfo['dtype'])
628 dtype=dsInfo['dtype'])
615 dtsets.append(ds)
629 dtsets.append(ds)
616 data.append((dsInfo['variable'], i))
630 data.append((dsInfo['variable'], i))
617 fp.flush()
631 fp.flush()
618
632
619 log.log('Creating file: {}'.format(fp.filename), self.name)
633 log.log('Creating file: {}'.format(fp.filename), self.name)
620
634
621 self.ds = dtsets
635 self.ds = dtsets
622 self.data = data
636 self.data = data
623 self.firsttime = True
637 self.firsttime = True
624 self.blockIndex = 0
638 self.blockIndex = 0
625 return
639 return
626
640
627 def putData(self):
641 def putData(self):
628
642
629 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
643 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
630 self.closeFile()
644 self.closeFile()
631 self.setNextFile()
645 self.setNextFile()
632
646
633 for i, ds in enumerate(self.ds):
647 for i, ds in enumerate(self.ds):
634 attr, ch = self.data[i]
648 attr, ch = self.data[i]
635 if ch == -1:
649 if ch == -1:
636 ds[self.blockIndex] = getattr(self.dataOut, attr)
650 ds[self.blockIndex] = getattr(self.dataOut, attr)
637 else:
651 else:
638 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
652 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
639
653
640 self.fp.flush()
654 self.fp.flush()
641 self.blockIndex += 1
655 self.blockIndex += 1
642 if self.blockIndex == 1:
656 if self.blockIndex == 1:
643 log.log('Block No. {}/{} --> {}'.format(self.blockIndex, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
657 log.log('Block No. {}/{} --> {}'.format(self.blockIndex, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
644 else:
658 else:
645 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
659 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
646 return
660 return
647
661
648 def closeFile(self):
662 def closeFile(self):
649
663
650 if self.blockIndex != self.blocksPerFile:
664 if self.blockIndex != self.blocksPerFile:
651 for ds in self.ds:
665 for ds in self.ds:
652 ds.resize(self.blockIndex, axis=0)
666 ds.resize(self.blockIndex, axis=0)
653
667
654 if self.fp:
668 if self.fp:
655 self.fp.flush()
669 self.fp.flush()
656 self.fp.close()
670 self.fp.close()
657
671
658 def close(self):
672 def close(self):
659
673
660 self.closeFile()
674 self.closeFile()
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
NO CONTENT: modified file
General Comments 0
You need to be logged in to leave comments. Login now