##// END OF EJS Templates
schain
RSS
GRABADO 1 SOLO CANAL H O V
avaldez -
r1596:5efbcbe7bd6e
parent child
Show More
Side by Side Unified
Context file:
r1596:5efbcbe7bd6e Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,739 +1,748
1 from email.utils import localtime
1 from email.utils import localtime
2 import os
2 import os
3 import time
3 import time
4 import datetime
4 import datetime
5
5
6 import numpy
6 import numpy
7 import h5py
7 import h5py
8
8
9 import schainpy.admin
9 import schainpy.admin
10 from schainpy.model.data.jrodata import *
10 from schainpy.model.data.jrodata import *
11 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
12 from schainpy.model.io.jroIO_base import *
12 from schainpy.model.io.jroIO_base import *
13 from schainpy.utils import log
13 from schainpy.utils import log
14
14
15
15
16 class HDFReader(Reader, ProcessingUnit):
16 class HDFReader(Reader, ProcessingUnit):
17 """Processing unit to read HDF5 format files
17 """Processing unit to read HDF5 format files
18
18
19 This unit reads HDF5 files created with `HDFWriter` operation contains
19 This unit reads HDF5 files created with `HDFWriter` operation contains
20 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 by default two groups Data and Metadata all variables would be saved as `dataOut`
21 attributes.
21 attributes.
22 It is possible to read any HDF5 file by given the structure in the `description`
22 It is possible to read any HDF5 file by given the structure in the `description`
23 parameter, also you can add extra values to metadata with the parameter `extras`.
23 parameter, also you can add extra values to metadata with the parameter `extras`.
24
24
25 Parameters:
25 Parameters:
26 -----------
26 -----------
27 path : str
27 path : str
28 Path where files are located.
28 Path where files are located.
29 startDate : date
29 startDate : date
30 Start date of the files
30 Start date of the files
31 endDate : list
31 endDate : list
32 End date of the files
32 End date of the files
33 startTime : time
33 startTime : time
34 Start time of the files
34 Start time of the files
35 endTime : time
35 endTime : time
36 End time of the files
36 End time of the files
37 description : dict, optional
37 description : dict, optional
38 Dictionary with the description of the HDF5 file
38 Dictionary with the description of the HDF5 file
39 extras : dict, optional
39 extras : dict, optional
40 Dictionary with extra metadata to be be added to `dataOut`
40 Dictionary with extra metadata to be be added to `dataOut`
41
41
42 Examples
42 Examples
43 --------
43 --------
44
44
45 desc = {
45 desc = {
46 'Data': {
46 'Data': {
47 'data_output': ['u', 'v', 'w'],
47 'data_output': ['u', 'v', 'w'],
48 'utctime': 'timestamps',
48 'utctime': 'timestamps',
49 } ,
49 } ,
50 'Metadata': {
50 'Metadata': {
51 'heightList': 'heights'
51 'heightList': 'heights'
52 }
52 }
53 }
53 }
54
54
55 desc = {
55 desc = {
56 'Data': {
56 'Data': {
57 'data_output': 'winds',
57 'data_output': 'winds',
58 'utctime': 'timestamps'
58 'utctime': 'timestamps'
59 },
59 },
60 'Metadata': {
60 'Metadata': {
61 'heightList': 'heights'
61 'heightList': 'heights'
62 }
62 }
63 }
63 }
64
64
65 extras = {
65 extras = {
66 'timeZone': 300
66 'timeZone': 300
67 }
67 }
68
68
69 reader = project.addReadUnit(
69 reader = project.addReadUnit(
70 name='HDFReader',
70 name='HDFReader',
71 path='/path/to/files',
71 path='/path/to/files',
72 startDate='2019/01/01',
72 startDate='2019/01/01',
73 endDate='2019/01/31',
73 endDate='2019/01/31',
74 startTime='00:00:00',
74 startTime='00:00:00',
75 endTime='23:59:59',
75 endTime='23:59:59',
76 # description=json.dumps(desc),
76 # description=json.dumps(desc),
77 # extras=json.dumps(extras),
77 # extras=json.dumps(extras),
78 )
78 )
79
79
80 """
80 """
81
81
82 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
83
83
84 def __init__(self):
84 def __init__(self):
85 ProcessingUnit.__init__(self)
85 ProcessingUnit.__init__(self)
86 self.dataOut = Parameters()
86 self.dataOut = Parameters()
87 self.ext = ".hdf5"
87 self.ext = ".hdf5"
88 self.optchar = "D"
88 self.optchar = "D"
89 self.meta = {}
89 self.meta = {}
90 self.data = {}
90 self.data = {}
91 self.open_file = h5py.File
91 self.open_file = h5py.File
92 self.open_mode = 'r'
92 self.open_mode = 'r'
93 self.description = {}
93 self.description = {}
94 self.extras = {}
94 self.extras = {}
95 self.filefmt = "*%Y%j***"
95 self.filefmt = "*%Y%j***"
96 self.folderfmt = "*%Y%j"
96 self.folderfmt = "*%Y%j"
97 self.utcoffset = 0
97 self.utcoffset = 0
98 self.filter = None
98 self.filter = None
99 self.dparam = None
99 self.dparam = None
100
100
101 def setup(self, **kwargs):
101 def setup(self, **kwargs):
102
102
103 self.set_kwargs(**kwargs)
103 self.set_kwargs(**kwargs)
104 if not self.ext.startswith('.'):
104 if not self.ext.startswith('.'):
105 self.ext = '.{}'.format(self.ext)
105 self.ext = '.{}'.format(self.ext)
106
106
107 if self.online:
107 if self.online:
108 log.log("Searching files in online mode...", self.name)
108 log.log("Searching files in online mode...", self.name)
109
109
110 for nTries in range(self.nTries):
110 for nTries in range(self.nTries):
111 fullpath = self.searchFilesOnLine(self.path, self.startDate,
111 fullpath = self.searchFilesOnLine(self.path, self.startDate,
112 self.endDate, self.expLabel, self.ext, self.walk,
112 self.endDate, self.expLabel, self.ext, self.walk,
113 self.filefmt, self.folderfmt,self.filter)
113 self.filefmt, self.folderfmt,self.filter)
114 try:
114 try:
115 fullpath = next(fullpath)
115 fullpath = next(fullpath)
116 except:
116 except:
117 fullpath = None
117 fullpath = None
118
118
119 if fullpath:
119 if fullpath:
120 break
120 break
121
121
122 log.warning(
122 log.warning(
123 'Waiting {} sec for a valid file in {}: try {} ...'.format(
123 'Waiting {} sec for a valid file in {}: try {} ...'.format(
124 self.delay, self.path, nTries + 1),
124 self.delay, self.path, nTries + 1),
125 self.name)
125 self.name)
126 time.sleep(self.delay)
126 time.sleep(self.delay)
127
127
128 if not(fullpath):
128 if not(fullpath):
129 raise schainpy.admin.SchainError(
129 raise schainpy.admin.SchainError(
130 'There isn\'t any valid file in {}'.format(self.path))
130 'There isn\'t any valid file in {}'.format(self.path))
131
131
132 pathname, filename = os.path.split(fullpath)
132 pathname, filename = os.path.split(fullpath)
133 self.year = int(filename[1:5])
133 self.year = int(filename[1:5])
134 self.doy = int(filename[5:8])
134 self.doy = int(filename[5:8])
135 self.set = int(filename[8:11]) - 1
135 self.set = int(filename[8:11]) - 1
136 else:
136 else:
137 log.log("Searching files in {}".format(self.path), self.name)
137 log.log("Searching files in {}".format(self.path), self.name)
138 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
138 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
139 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt,self.filter)
139 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt,self.filter)
140
140
141 self.setNextFile()
141 self.setNextFile()
142
142
143 return
143 return
144
144
145 def readFirstHeader(self):
145 def readFirstHeader(self):
146 '''Read metadata and data'''
146 '''Read metadata and data'''
147
147
148 self.__readMetadata()
148 self.__readMetadata()
149 self.__readData()
149 self.__readData()
150 self.__setBlockList()
150 self.__setBlockList()
151
151
152 if 'type' in self.meta:
152 if 'type' in self.meta:
153 self.dataOut = eval(self.meta['type'])()
153 self.dataOut = eval(self.meta['type'])()
154
154
155 if self.dparam:
155 if self.dparam:
156 setattr(self.dataOut, "dparam", 1)
156 setattr(self.dataOut, "dparam", 1)
157
157
158 for attr in self.meta:
158 for attr in self.meta:
159 setattr(self.dataOut, attr, self.meta[attr])
159 setattr(self.dataOut, attr, self.meta[attr])
160
160
161 self.blockIndex = 0
161 self.blockIndex = 0
162
162
163 return
163 return
164
164
165 def __setBlockList(self):
165 def __setBlockList(self):
166 '''
166 '''
167 Selects the data within the times defined
167 Selects the data within the times defined
168
168
169 self.fp
169 self.fp
170 self.startTime
170 self.startTime
171 self.endTime
171 self.endTime
172 self.blockList
172 self.blockList
173 self.blocksPerFile
173 self.blocksPerFile
174
174
175 '''
175 '''
176
176
177 startTime = self.startTime
177 startTime = self.startTime
178 endTime = self.endTime
178 endTime = self.endTime
179 thisUtcTime = self.data['utctime'] + self.utcoffset
179 thisUtcTime = self.data['utctime'] + self.utcoffset
180 try:
180 try:
181 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
181 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
182 except:
182 except:
183 self.interval = 0
183 self.interval = 0
184 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
184 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
185
185
186 thisDate = thisDatetime.date()
186 thisDate = thisDatetime.date()
187 thisTime = thisDatetime.time()
187 thisTime = thisDatetime.time()
188
188
189 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
189 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
190 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
190 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
191
191
192 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
192 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
193
193
194 self.blockList = ind
194 self.blockList = ind
195 self.blocksPerFile = len(ind)
195 self.blocksPerFile = len(ind)
196 return
196 return
197
197
198 def __readMetadata(self):
198 def __readMetadata(self):
199 '''
199 '''
200 Reads Metadata
200 Reads Metadata
201 '''
201 '''
202
202
203 meta = {}
203 meta = {}
204
204
205 if self.description:
205 if self.description:
206 for key, value in self.description['Metadata'].items():
206 for key, value in self.description['Metadata'].items():
207 meta[key] = self.fp[value][()]
207 meta[key] = self.fp[value][()]
208 else:
208 else:
209 grp = self.fp['Metadata']
209 grp = self.fp['Metadata']
210 for name in grp:
210 for name in grp:
211 meta[name] = grp[name][()]
211 meta[name] = grp[name][()]
212
212
213 if self.extras:
213 if self.extras:
214 for key, value in self.extras.items():
214 for key, value in self.extras.items():
215 meta[key] = value
215 meta[key] = value
216 self.meta = meta
216 self.meta = meta
217
217
218 return
218 return
219
219
220 def __readData(self):
220 def __readData(self):
221
221
222 data = {}
222 data = {}
223
223
224 if self.description:
224 if self.description:
225 for key, value in self.description['Data'].items():
225 for key, value in self.description['Data'].items():
226 if isinstance(value, str):
226 if isinstance(value, str):
227 if isinstance(self.fp[value], h5py.Dataset):
227 if isinstance(self.fp[value], h5py.Dataset):
228 data[key] = self.fp[value][()]
228 data[key] = self.fp[value][()]
229 elif isinstance(self.fp[value], h5py.Group):
229 elif isinstance(self.fp[value], h5py.Group):
230 array = []
230 array = []
231 for ch in self.fp[value]:
231 for ch in self.fp[value]:
232 array.append(self.fp[value][ch][()])
232 array.append(self.fp[value][ch][()])
233 data[key] = numpy.array(array)
233 data[key] = numpy.array(array)
234 elif isinstance(value, list):
234 elif isinstance(value, list):
235 array = []
235 array = []
236 for ch in value:
236 for ch in value:
237 array.append(self.fp[ch][()])
237 array.append(self.fp[ch][()])
238 data[key] = numpy.array(array)
238 data[key] = numpy.array(array)
239 else:
239 else:
240 grp = self.fp['Data']
240 grp = self.fp['Data']
241 for name in grp:
241 for name in grp:
242 if isinstance(grp[name], h5py.Dataset):
242 if isinstance(grp[name], h5py.Dataset):
243 array = grp[name][()]
243 array = grp[name][()]
244 elif isinstance(grp[name], h5py.Group):
244 elif isinstance(grp[name], h5py.Group):
245 array = []
245 array = []
246 for ch in grp[name]:
246 for ch in grp[name]:
247 array.append(grp[name][ch][()])
247 array.append(grp[name][ch][()])
248 array = numpy.array(array)
248 array = numpy.array(array)
249 else:
249 else:
250 log.warning('Unknown type: {}'.format(name))
250 log.warning('Unknown type: {}'.format(name))
251
251
252 if name in self.description:
252 if name in self.description:
253 key = self.description[name]
253 key = self.description[name]
254 else:
254 else:
255 key = name
255 key = name
256 data[key] = array
256 data[key] = array
257
257
258 self.data = data
258 self.data = data
259 return
259 return
260
260
261 def getData(self):
261 def getData(self):
262
262
263 for attr in self.data:
263 for attr in self.data:
264 if self.data[attr].ndim == 1:
264 if self.data[attr].ndim == 1:
265 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
265 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
266 else:
266 else:
267 if self.dparam:
267 if self.dparam:
268 setattr(self.dataOut, attr, self.data[attr])
268 setattr(self.dataOut, attr, self.data[attr])
269 else:
269 else:
270 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
270 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
271
271
272 self.dataOut.flagNoData = False
272 self.dataOut.flagNoData = False
273 self.blockIndex += 1
273 self.blockIndex += 1
274
274
275 log.log("Block No. {}/{} -> {}".format(
275 log.log("Block No. {}/{} -> {}".format(
276 self.blockIndex,
276 self.blockIndex,
277 self.blocksPerFile,
277 self.blocksPerFile,
278 self.dataOut.datatime.ctime()), self.name)
278 self.dataOut.datatime.ctime()), self.name)
279
279
280 return
280 return
281
281
282 def run(self, **kwargs):
282 def run(self, **kwargs):
283
283
284 if not(self.isConfig):
284 if not(self.isConfig):
285 self.setup(**kwargs)
285 self.setup(**kwargs)
286 self.isConfig = True
286 self.isConfig = True
287
287
288 if self.blockIndex == self.blocksPerFile:
288 if self.blockIndex == self.blocksPerFile:
289 self.setNextFile()
289 self.setNextFile()
290
290
291 self.getData()
291 self.getData()
292
292
293 return
293 return
294
294
295 @MPDecorator
295 @MPDecorator
296 class HDFWriter(Operation):
296 class HDFWriter(Operation):
297 """Operation to write HDF5 files.
297 """Operation to write HDF5 files.
298
298
299 The HDF5 file contains by default two groups Data and Metadata where
299 The HDF5 file contains by default two groups Data and Metadata where
300 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
300 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
301 parameters, data attributes are normaly time dependent where the metadata
301 parameters, data attributes are normaly time dependent where the metadata
302 are not.
302 are not.
303 It is possible to customize the structure of the HDF5 file with the
303 It is possible to customize the structure of the HDF5 file with the
304 optional description parameter see the examples.
304 optional description parameter see the examples.
305
305
306 Parameters:
306 Parameters:
307 -----------
307 -----------
308 path : str
308 path : str
309 Path where files will be saved.
309 Path where files will be saved.
310 blocksPerFile : int
310 blocksPerFile : int
311 Number of blocks per file
311 Number of blocks per file
312 metadataList : list
312 metadataList : list
313 List of the dataOut attributes that will be saved as metadata
313 List of the dataOut attributes that will be saved as metadata
314 dataList : int
314 dataList : int
315 List of the dataOut attributes that will be saved as data
315 List of the dataOut attributes that will be saved as data
316 setType : bool
316 setType : bool
317 If True the name of the files corresponds to the timestamp of the data
317 If True the name of the files corresponds to the timestamp of the data
318 description : dict, optional
318 description : dict, optional
319 Dictionary with the desired description of the HDF5 file
319 Dictionary with the desired description of the HDF5 file
320
320
321 Examples
321 Examples
322 --------
322 --------
323
323
324 desc = {
324 desc = {
325 'data_output': {'winds': ['z', 'w', 'v']},
325 'data_output': {'winds': ['z', 'w', 'v']},
326 'utctime': 'timestamps',
326 'utctime': 'timestamps',
327 'heightList': 'heights'
327 'heightList': 'heights'
328 }
328 }
329 desc = {
329 desc = {
330 'data_output': ['z', 'w', 'v'],
330 'data_output': ['z', 'w', 'v'],
331 'utctime': 'timestamps',
331 'utctime': 'timestamps',
332 'heightList': 'heights'
332 'heightList': 'heights'
333 }
333 }
334 desc = {
334 desc = {
335 'Data': {
335 'Data': {
336 'data_output': 'winds',
336 'data_output': 'winds',
337 'utctime': 'timestamps'
337 'utctime': 'timestamps'
338 },
338 },
339 'Metadata': {
339 'Metadata': {
340 'heightList': 'heights'
340 'heightList': 'heights'
341 }
341 }
342 }
342 }
343
343
344 writer = proc_unit.addOperation(name='HDFWriter')
344 writer = proc_unit.addOperation(name='HDFWriter')
345 writer.addParameter(name='path', value='/path/to/file')
345 writer.addParameter(name='path', value='/path/to/file')
346 writer.addParameter(name='blocksPerFile', value='32')
346 writer.addParameter(name='blocksPerFile', value='32')
347 writer.addParameter(name='metadataList', value='heightList,timeZone')
347 writer.addParameter(name='metadataList', value='heightList,timeZone')
348 writer.addParameter(name='dataList',value='data_output,utctime')
348 writer.addParameter(name='dataList',value='data_output,utctime')
349 # writer.addParameter(name='description',value=json.dumps(desc))
349 # writer.addParameter(name='description',value=json.dumps(desc))
350
350
351 """
351 """
352
352
353 ext = ".hdf5"
353 ext = ".hdf5"
354 optchar = "D"
354 optchar = "D"
355 filename = None
355 filename = None
356 path = None
356 path = None
357 setFile = None
357 setFile = None
358 fp = None
358 fp = None
359 firsttime = True
359 firsttime = True
360 #Configurations
360 #Configurations
361 blocksPerFile = None
361 blocksPerFile = None
362 blockIndex = None
362 blockIndex = None
363 dataOut = None
363 dataOut = None
364 #Data Arrays
364 #Data Arrays
365 dataList = None
365 dataList = None
366 metadataList = None
366 metadataList = None
367 currentDay = None
367 currentDay = None
368 lastTime = None
368 lastTime = None
369 last_Azipos = None
369 last_Azipos = None
370 last_Elepos = None
370 last_Elepos = None
371 mode = None
371 mode = None
372 #-----------------------
372 #-----------------------
373 Typename = None
373 Typename = None
374 mask = False
374 mask = False
375 setChannel = None
375
376
376 def __init__(self):
377 def __init__(self):
377
378
378 Operation.__init__(self)
379 Operation.__init__(self)
379 return
380 return
380
381
381 def set_kwargs(self, **kwargs):
382 def set_kwargs(self, **kwargs):
382
383
383 for key, value in kwargs.items():
384 for key, value in kwargs.items():
384 setattr(self, key, value)
385 setattr(self, key, value)
385
386
386 def set_kwargs_obj(self,obj, **kwargs):
387 def set_kwargs_obj(self,obj, **kwargs):
387
388
388 for key, value in kwargs.items():
389 for key, value in kwargs.items():
389 setattr(obj, key, value)
390 setattr(obj, key, value)
390
391
391 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None,type_data=None, localtime=True, **kwargs):
392 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None,type_data=None, localtime=True,setChannel=None, **kwargs):
392 self.path = path
393 self.path = path
393 self.blocksPerFile = blocksPerFile
394 self.blocksPerFile = blocksPerFile
394 self.metadataList = metadataList
395 self.metadataList = metadataList
395 self.dataList = [s.strip() for s in dataList]
396 self.dataList = [s.strip() for s in dataList]
397 self.setChannel = setChannel
396 self.setType = setType
398 self.setType = setType
397 if self.setType == "weather":
399 if self.setType == "weather":
398 self.set_kwargs(**kwargs)
400 self.set_kwargs(**kwargs)
399 self.set_kwargs_obj(self.dataOut,**kwargs)
401 self.set_kwargs_obj(self.dataOut,**kwargs)
400 self.weather_vars = {
402 self.weather_vars = {
401 'S' : 0,
403 'S' : 0,
402 'V' : 1,
404 'V' : 1,
403 'W' : 2,
405 'W' : 2,
404 'SNR' : 3,
406 'SNR' : 3,
405 'Z' : 4,
407 'Z' : 4,
406 'D' : 5,
408 'D' : 5,
407 'P' : 6,
409 'P' : 6,
408 'R' : 7,
410 'R' : 7,
409 }
411 }
410
412
411 if localtime:
413 if localtime:
412 self.getDateTime = datetime.datetime.fromtimestamp
414 self.getDateTime = datetime.datetime.fromtimestamp
413 else:
415 else:
414 self.getDateTime = datetime.datetime.utcfromtimestamp
416 self.getDateTime = datetime.datetime.utcfromtimestamp
415
417
416 self.description = description
418 self.description = description
417 self.type_data=type_data
419 self.type_data=type_data
418
420
419 if self.metadataList is None:
421 if self.metadataList is None:
420 self.metadataList = self.dataOut.metadata_list
422 self.metadataList = self.dataOut.metadata_list
421
423
422 dsList = []
424 dsList = []
423
425
424 for i in range(len(self.dataList)):
426 for i in range(len(self.dataList)):
425 dsDict = {}
427 dsDict = {}
426 if hasattr(self.dataOut, self.dataList[i]):
428 if hasattr(self.dataOut, self.dataList[i]):
427 dataAux = getattr(self.dataOut, self.dataList[i])
429 dataAux = getattr(self.dataOut, self.dataList[i])
428 if self.setType == 'weather' and self.dataList[i] == 'data_param':
430 if self.setType == 'weather' and self.dataList[i] == 'data_param':
429 dataAux = dataAux[:,self.weather_vars[self.weather_var],:]
431 if self.setChannel is None:
432 dataAux = dataAux[:,self.weather_vars[self.weather_var],:]
433 else:
434 dataAux = dataAux[self.setChannel,self.weather_vars[self.weather_var],:]
435 dataAux = numpy.reshape(dataAux,(1,dataAux.shape[0],dataAux.shape[1]))
430 dsDict['variable'] = self.dataList[i]
436 dsDict['variable'] = self.dataList[i]
431 else:
437 else:
432 log.warning('Attribute {} not found in dataOut'.format(self.dataList[i]), self.name)
438 log.warning('Attribute {} not found in dataOut'.format(self.dataList[i]), self.name)
433 continue
439 continue
434
440
435 if dataAux is None:
441 if dataAux is None:
436 continue
442 continue
437 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
443 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
438 dsDict['nDim'] = 0
444 dsDict['nDim'] = 0
439 else:
445 else:
440 dsDict['nDim'] = len(dataAux.shape)
446 dsDict['nDim'] = len(dataAux.shape)
441 dsDict['shape'] = dataAux.shape
447 dsDict['shape'] = dataAux.shape
442 dsDict['dsNumber'] = dataAux.shape[0]
448 dsDict['dsNumber'] = dataAux.shape[0]
443 dsDict['dtype'] = dataAux.dtype
449 dsDict['dtype'] = dataAux.dtype
444 dsList.append(dsDict)
450 dsList.append(dsDict)
445
451
446 self.dsList = dsList
452 self.dsList = dsList
447 self.currentDay = self.dataOut.datatime.date()
453 self.currentDay = self.dataOut.datatime.date()
448
454
449 def timeFlag(self):
455 def timeFlag(self):
450 currentTime = self.dataOut.utctime
456 currentTime = self.dataOut.utctime
451 dt = self.getDateTime(currentTime)
457 dt = self.getDateTime(currentTime)
452
458
453 dataDay = int(dt.strftime('%j'))
459 dataDay = int(dt.strftime('%j'))
454
460
455 if self.lastTime is None:
461 if self.lastTime is None:
456 self.lastTime = currentTime
462 self.lastTime = currentTime
457 self.currentDay = dataDay
463 self.currentDay = dataDay
458 return False
464 return False
459
465
460 timeDiff = currentTime - self.lastTime
466 timeDiff = currentTime - self.lastTime
461
467
462 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
468 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
463 if dataDay != self.currentDay:
469 if dataDay != self.currentDay:
464 self.currentDay = dataDay
470 self.currentDay = dataDay
465 return True
471 return True
466 elif timeDiff > 3*60*60:
472 elif timeDiff > 3*60*60:
467 self.lastTime = currentTime
473 self.lastTime = currentTime
468 return True
474 return True
469 else:
475 else:
470 self.lastTime = currentTime
476 self.lastTime = currentTime
471 return False
477 return False
472
478
473 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
479 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
474 dataList=[], setType=None, description={}, mode= None,
480 dataList=[], setType=None, description={}, mode= None,
475 type_data=None, Reset = False, localtime=True, **kwargs):
481 type_data=None, Reset = False, localtime=True, **kwargs):
476
482
477 if Reset:
483 if Reset:
478 self.isConfig = False
484 self.isConfig = False
479 self.closeFile()
485 self.closeFile()
480 self.lastTime = None
486 self.lastTime = None
481 self.blockIndex = 0
487 self.blockIndex = 0
482
488
483 self.dataOut = dataOut
489 self.dataOut = dataOut
484 self.mode = mode
490 self.mode = mode
485
491
486 if not(self.isConfig):
492 if not(self.isConfig):
487 self.setup(path=path, blocksPerFile=blocksPerFile,
493 self.setup(path=path, blocksPerFile=blocksPerFile,
488 metadataList=metadataList, dataList=dataList,
494 metadataList=metadataList, dataList=dataList,
489 setType=setType, description=description,type_data=type_data,
495 setType=setType, description=description,type_data=type_data,
490 localtime=localtime, **kwargs)
496 localtime=localtime, **kwargs)
491
497
492 self.isConfig = True
498 self.isConfig = True
493 self.setNextFile()
499 self.setNextFile()
494
500
495 self.putData()
501 self.putData()
496 return
502 return
497
503
498 def setNextFile(self):
504 def setNextFile(self):
499
505
500 ext = self.ext
506 ext = self.ext
501 path = self.path
507 path = self.path
502 setFile = self.setFile
508 setFile = self.setFile
503
509
504 dt = self.getDateTime(self.dataOut.utctime)
510 dt = self.getDateTime(self.dataOut.utctime)
505
511
506 if self.setType == 'weather':
512 if self.setType == 'weather':
507 subfolder = dt.strftime('%Y-%m-%dT%H-00-00')
513 subfolder = dt.strftime('%Y-%m-%dT%H-00-00')
508 subfolder = ''
514 subfolder = ''
509 else:
515 else:
510 subfolder = dt.strftime('d%Y%j')
516 subfolder = dt.strftime('d%Y%j')
511
517
512 fullpath = os.path.join(path, subfolder)
518 fullpath = os.path.join(path, subfolder)
513
519
514 if os.path.exists(fullpath):
520 if os.path.exists(fullpath):
515 filesList = os.listdir(fullpath)
521 filesList = os.listdir(fullpath)
516 filesList = [k for k in filesList if k.startswith(self.optchar)]
522 filesList = [k for k in filesList if k.startswith(self.optchar)]
517 if len( filesList ) > 0:
523 if len( filesList ) > 0:
518 filesList = sorted(filesList, key=str.lower)
524 filesList = sorted(filesList, key=str.lower)
519 filen = filesList[-1]
525 filen = filesList[-1]
520 # el filename debera tener el siguiente formato
526 # el filename debera tener el siguiente formato
521 # 0 1234 567 89A BCDE (hex)
527 # 0 1234 567 89A BCDE (hex)
522 # x YYYY DDD SSS .ext
528 # x YYYY DDD SSS .ext
523 if isNumber(filen[8:11]):
529 if isNumber(filen[8:11]):
524 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
530 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
525 else:
531 else:
526 setFile = -1
532 setFile = -1
527 else:
533 else:
528 setFile = -1 #inicializo mi contador de seteo
534 setFile = -1 #inicializo mi contador de seteo
529 else:
535 else:
530 os.makedirs(fullpath)
536 os.makedirs(fullpath)
531 setFile = -1 #inicializo mi contador de seteo
537 setFile = -1 #inicializo mi contador de seteo
532
538
533 if self.setType is None:
539 if self.setType is None:
534 setFile += 1
540 setFile += 1
535 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
541 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
536 dt.year,
542 dt.year,
537 int(dt.strftime('%j')),
543 int(dt.strftime('%j')),
538 setFile,
544 setFile,
539 ext )
545 ext )
540 elif self.setType == "weather":
546 elif self.setType == "weather":
541
547
542 #SOPHY_20200505_140215_E10.0_Z.h5
548 #SOPHY_20200505_140215_E10.0_Z.h5
543 #SOPHY_20200505_140215_A40.0_Z.h5
549 #SOPHY_20200505_140215_A40.0_Z.h5
544 if self.dataOut.flagMode == 1: #'AZI' #PPI
550 if self.dataOut.flagMode == 1: #'AZI' #PPI
545 ang_type = 'EL'
551 ang_type = 'EL'
546 mode_type = 'PPI'
552 mode_type = 'PPI'
547 len_aux = int(self.dataOut.data_ele.shape[0]/4)
553 len_aux = int(self.dataOut.data_ele.shape[0]/4)
548 mean = numpy.mean(self.dataOut.data_ele[len_aux:-len_aux])
554 mean = numpy.mean(self.dataOut.data_ele[len_aux:-len_aux])
549 ang_ = round(mean,1)
555 ang_ = round(mean,1)
550 elif self.dataOut.flagMode == 0: #'ELE' #RHI
556 elif self.dataOut.flagMode == 0: #'ELE' #RHI
551 ang_type = 'AZ'
557 ang_type = 'AZ'
552 mode_type = 'RHI'
558 mode_type = 'RHI'
553 len_aux = int(self.dataOut.data_azi.shape[0]/4)
559 len_aux = int(self.dataOut.data_azi.shape[0]/4)
554 mean = numpy.mean(self.dataOut.data_azi[len_aux:-len_aux])
560 mean = numpy.mean(self.dataOut.data_azi[len_aux:-len_aux])
555 ang_ = round(mean,1)
561 ang_ = round(mean,1)
556
562
557 file = '%s_%2.2d%2.2d%2.2d_%2.2d%2.2d%2.2d_%s%2.1f_%s%s' % (
563 file = '%s_%2.2d%2.2d%2.2d_%2.2d%2.2d%2.2d_%s%2.1f_%s%s' % (
558 'SOPHY',
564 'SOPHY',
559 dt.year,
565 dt.year,
560 dt.month,
566 dt.month,
561 dt.day,
567 dt.day,
562 dt.hour,
568 dt.hour,
563 dt.minute,
569 dt.minute,
564 dt.second,
570 dt.second,
565 ang_type[0],
571 ang_type[0],
566 ang_,
572 ang_,
567 self.weather_var,
573 self.weather_var,
568 ext )
574 ext )
569 subfolder = '{}_{}_{}_{:2.1f}'.format(self.weather_var, mode_type, ang_type, ang_)
575 subfolder = '{}_{}_{}_{:2.1f}'.format(self.weather_var, mode_type, ang_type, ang_)
570 fullpath = os.path.join(path, subfolder)
576 fullpath = os.path.join(path, subfolder)
571 if not os.path.exists(fullpath):
577 if not os.path.exists(fullpath):
572 os.makedirs(fullpath)
578 os.makedirs(fullpath)
573 else:
579 else:
574 setFile = dt.hour*60+dt.minute
580 setFile = dt.hour*60+dt.minute
575 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
581 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
576 dt.year,
582 dt.year,
577 int(dt.strftime('%j')),
583 int(dt.strftime('%j')),
578 setFile,
584 setFile,
579 ext )
585 ext )
580
586
581 self.filename = os.path.join( path, subfolder, file )
587 self.filename = os.path.join( path, subfolder, file )
582
588
583 self.fp = h5py.File(self.filename, 'w')
589 self.fp = h5py.File(self.filename, 'w')
584 #write metadata
590 #write metadata
585 self.writeMetadata(self.fp)
591 self.writeMetadata(self.fp)
586 #Write data
592 #Write data
587 self.writeData(self.fp)
593 self.writeData(self.fp)
588
594
589 def getLabel(self, name, x=None):
595 def getLabel(self, name, x=None):
590
596
591 if x is None:
597 if x is None:
592 if 'Data' in self.description:
598 if 'Data' in self.description:
593 data = self.description['Data']
599 data = self.description['Data']
594 if 'Metadata' in self.description:
600 if 'Metadata' in self.description:
595 data.update(self.description['Metadata'])
601 data.update(self.description['Metadata'])
596 else:
602 else:
597 data = self.description
603 data = self.description
598 if name in data:
604 if name in data:
599 if isinstance(data[name], str):
605 if isinstance(data[name], str):
600 return data[name]
606 return data[name]
601 elif isinstance(data[name], list):
607 elif isinstance(data[name], list):
602 return None
608 return None
603 elif isinstance(data[name], dict):
609 elif isinstance(data[name], dict):
604 for key, value in data[name].items():
610 for key, value in data[name].items():
605 return key
611 return key
606 return name
612 return name
607 else:
613 else:
608 if 'Data' in self.description:
614 if 'Data' in self.description:
609 data = self.description['Data']
615 data = self.description['Data']
610 if 'Metadata' in self.description:
616 if 'Metadata' in self.description:
611 data.update(self.description['Metadata'])
617 data.update(self.description['Metadata'])
612 else:
618 else:
613 data = self.description
619 data = self.description
614 if name in data:
620 if name in data:
615 if isinstance(data[name], list):
621 if isinstance(data[name], list):
616 return data[name][x]
622 return data[name][x]
617 elif isinstance(data[name], dict):
623 elif isinstance(data[name], dict):
618 for key, value in data[name].items():
624 for key, value in data[name].items():
619 return value[x]
625 return value[x]
620 if 'cspc' in name:
626 if 'cspc' in name:
621 return 'pair{:02d}'.format(x)
627 return 'pair{:02d}'.format(x)
622 else:
628 else:
623 return 'channel{:02d}'.format(x)
629 return 'channel{:02d}'.format(x)
624
630
625 def writeMetadata(self, fp):
631 def writeMetadata(self, fp):
626
632
627 if self.description:
633 if self.description:
628 if 'Metadata' in self.description:
634 if 'Metadata' in self.description:
629 grp = fp.create_group('Metadata')
635 grp = fp.create_group('Metadata')
630 else:
636 else:
631 grp = fp
637 grp = fp
632 else:
638 else:
633 grp = fp.create_group('Metadata')
639 grp = fp.create_group('Metadata')
634
640
635 for i in range(len(self.metadataList)):
641 for i in range(len(self.metadataList)):
636 if not hasattr(self.dataOut, self.metadataList[i]):
642 if not hasattr(self.dataOut, self.metadataList[i]):
637 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
643 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
638 continue
644 continue
639 value = getattr(self.dataOut, self.metadataList[i])
645 value = getattr(self.dataOut, self.metadataList[i])
640 if isinstance(value, bool):
646 if isinstance(value, bool):
641 if value is True:
647 if value is True:
642 value = 1
648 value = 1
643 else:
649 else:
644 value = 0
650 value = 0
645 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
651 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
646 return
652 return
647
653
648 def writeData(self, fp):
654 def writeData(self, fp):
649
655
650 if self.description:
656 if self.description:
651 if 'Data' in self.description:
657 if 'Data' in self.description:
652 grp = fp.create_group('Data')
658 grp = fp.create_group('Data')
653 else:
659 else:
654 grp = fp
660 grp = fp
655 else:
661 else:
656 grp = fp.create_group('Data')
662 grp = fp.create_group('Data')
657
663
658 dtsets = []
664 dtsets = []
659 data = []
665 data = []
660
666
661 for dsInfo in self.dsList:
667 for dsInfo in self.dsList:
662
668
663 if dsInfo['nDim'] == 0:
669 if dsInfo['nDim'] == 0:
664 ds = grp.create_dataset(
670 ds = grp.create_dataset(
665 self.getLabel(dsInfo['variable']),
671 self.getLabel(dsInfo['variable']),
666 (self.blocksPerFile, ),
672 (self.blocksPerFile, ),
667 chunks=True,
673 chunks=True,
668 dtype=numpy.float64)
674 dtype=numpy.float64)
669 dtsets.append(ds)
675 dtsets.append(ds)
670 data.append((dsInfo['variable'], -1))
676 data.append((dsInfo['variable'], -1))
671 else:
677 else:
672 label = self.getLabel(dsInfo['variable'])
678 label = self.getLabel(dsInfo['variable'])
673 if label is not None:
679 if label is not None:
674 sgrp = grp.create_group(label)
680 sgrp = grp.create_group(label)
675 else:
681 else:
676 sgrp = grp
682 sgrp = grp
677 if self.blocksPerFile == 1:
683 if self.blocksPerFile == 1:
678 shape = dsInfo['shape'][1:]
684 shape = dsInfo['shape'][1:]
679 else:
685 else:
680 shape = (self.blocksPerFile, ) + dsInfo['shape'][1:]
686 shape = (self.blocksPerFile, ) + dsInfo['shape'][1:]
681 for i in range(dsInfo['dsNumber']):
687 for i in range(dsInfo['dsNumber']):
688 if dsInfo['dsNumber']==1:
689 if self.setChannel==1:
690 i=1
682 ds = sgrp.create_dataset(
691 ds = sgrp.create_dataset(
683 self.getLabel(dsInfo['variable'], i),
692 self.getLabel(dsInfo['variable'], i),
684 shape,
693 shape,
685 chunks=True,
694 chunks=True,
686 dtype=dsInfo['dtype'],
695 dtype=dsInfo['dtype'],
687 compression='gzip',
696 compression='gzip',
688 )
697 )
689 dtsets.append(ds)
698 dtsets.append(ds)
690 data.append((dsInfo['variable'], i))
699 data.append((dsInfo['variable'], i))
691 fp.flush()
700 fp.flush()
692
701
693 log.log('Creating file: {}'.format(fp.filename), self.name)
702 log.log('Creating file: {}'.format(fp.filename), self.name)
694
703
695 self.ds = dtsets
704 self.ds = dtsets
696 self.data = data
705 self.data = data
697 self.firsttime = True
706 self.firsttime = True
698 self.blockIndex = 0
707 self.blockIndex = 0
699 return
708 return
700
709
701 def putData(self):
710 def putData(self):
702
711
703 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
712 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
704 self.closeFile()
713 self.closeFile()
705 self.setNextFile()
714 self.setNextFile()
706
715
707 for i, ds in enumerate(self.ds):
716 for i, ds in enumerate(self.ds):
708 attr, ch = self.data[i]
717 attr, ch = self.data[i]
709 if ch == -1:
718 if ch == -1:
710 ds[self.blockIndex] = getattr(self.dataOut, attr)
719 ds[self.blockIndex] = getattr(self.dataOut, attr)
711 else:
720 else:
712 if self.blocksPerFile == 1:
721 if self.blocksPerFile == 1:
713 mask = self.dataOut.data_param[:,3,:][ch] < self.mask
722 mask = self.dataOut.data_param[:,3,:][ch] < self.mask
714 tmp = getattr(self.dataOut, attr)[:,self.weather_vars[self.weather_var],:][ch]
723 tmp = getattr(self.dataOut, attr)[:,self.weather_vars[self.weather_var],:][ch]
715 if self.mask:
724 if self.mask:
716 tmp[mask] = numpy.nan
725 tmp[mask] = numpy.nan
717 ds[:] = tmp
726 ds[:] = tmp
718 else:
727 else:
719 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
728 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
720
729
721 self.fp.flush()
730 self.fp.flush()
722 self.blockIndex += 1
731 self.blockIndex += 1
723 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
732 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
724
733
725 return
734 return
726
735
727 def closeFile(self):
736 def closeFile(self):
728
737
729 if self.blockIndex != self.blocksPerFile:
738 if self.blockIndex != self.blocksPerFile:
730 for ds in self.ds:
739 for ds in self.ds:
731 ds.resize(self.blockIndex, axis=0)
740 ds.resize(self.blockIndex, axis=0)
732
741
733 if self.fp:
742 if self.fp:
734 self.fp.flush()
743 self.fp.flush()
735 self.fp.close()
744 self.fp.close()
736
745
737 def close(self):
746 def close(self):
738
747
739 self.closeFile()
748 self.closeFile()
General Comments 0
You need to be logged in to leave comments. Login now