##// END OF EJS Templates
schain
RSS
v3-devel similar to master with these changes.
imanay -
r1702:eaa588a03960
parent child
Show More
Side by Side Unified
Context file:
r1702:eaa588a03960 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,887 +1,897
1 import os
1 import os
2 import time
2 import time
3 import datetime
3 import datetime
4
4
5 import numpy
5 import numpy
6 import h5py
6 import h5py
7
7
8 import schainpy.admin
8 import schainpy.admin
9 from schainpy.model.data.jrodata import *
9 from schainpy.model.data.jrodata import *
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 from schainpy.utils import log
12 from schainpy.utils import log
13
13
14
14
15 class HDFReader(Reader, ProcessingUnit):
15 class HDFReader(Reader, ProcessingUnit):
16 """Processing unit to read HDF5 format files
16 """Processing unit to read HDF5 format files
17
17
18 This unit reads HDF5 files created with `HDFWriter` operation contains
18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 attributes.
20 attributes.
21 It is possible to read any HDF5 file by given the structure in the `description`
21 It is possible to read any HDF5 file by given the structure in the `description`
22 parameter, also you can add extra values to metadata with the parameter `extras`.
22 parameter, also you can add extra values to metadata with the parameter `extras`.
23
23
24 Parameters:
24 Parameters:
25 -----------
25 -----------
26 path : str
26 path : str
27 Path where files are located.
27 Path where files are located.
28 startDate : date
28 startDate : date
29 Start date of the files
29 Start date of the files
30 endDate : list
30 endDate : list
31 End date of the files
31 End date of the files
32 startTime : time
32 startTime : time
33 Start time of the files
33 Start time of the files
34 endTime : time
34 endTime : time
35 End time of the files
35 End time of the files
36 description : dict, optional
36 description : dict, optional
37 Dictionary with the description of the HDF5 file
37 Dictionary with the description of the HDF5 file
38 extras : dict, optional
38 extras : dict, optional
39 Dictionary with extra metadata to be be added to `dataOut`
39 Dictionary with extra metadata to be be added to `dataOut`
40
40
41 Examples
41 Examples
42 --------
42 --------
43
43
44 desc = {
44 desc = {
45 'Data': {
45 'Data': {
46 'data_output': ['u', 'v', 'w'],
46 'data_output': ['u', 'v', 'w'],
47 'utctime': 'timestamps',
47 'utctime': 'timestamps',
48 } ,
48 } ,
49 'Metadata': {
49 'Metadata': {
50 'heightList': 'heights'
50 'heightList': 'heights'
51 }
51 }
52 }
52 }
53
53
54 desc = {
54 desc = {
55 'Data': {
55 'Data': {
56 'data_output': 'winds',
56 'data_output': 'winds',
57 'utctime': 'timestamps'
57 'utctime': 'timestamps'
58 },
58 },
59 'Metadata': {
59 'Metadata': {
60 'heightList': 'heights'
60 'heightList': 'heights'
61 }
61 }
62 }
62 }
63
63
64 extras = {
64 extras = {
65 'timeZone': 300
65 'timeZone': 300
66 }
66 }
67
67
68 reader = project.addReadUnit(
68 reader = project.addReadUnit(
69 name='HDFReader',
69 name='HDFReader',
70 path='/path/to/files',
70 path='/path/to/files',
71 startDate='2019/01/01',
71 startDate='2019/01/01',
72 endDate='2019/01/31',
72 endDate='2019/01/31',
73 startTime='00:00:00',
73 startTime='00:00:00',
74 endTime='23:59:59',
74 endTime='23:59:59',
75 # description=json.dumps(desc),
75 # description=json.dumps(desc),
76 # extras=json.dumps(extras),
76 # extras=json.dumps(extras),
77 )
77 )
78
78
79 """
79 """
80
80
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82
82
83 def __init__(self):
83 def __init__(self):
84 ProcessingUnit.__init__(self)
84 ProcessingUnit.__init__(self)
85 self.dataOut = Parameters()
85 self.dataOut = Parameters()
86 self.ext = ".hdf5"
86 self.ext = ".hdf5"
87 self.optchar = "D"
87 self.optchar = "D"
88 self.meta = {}
88 self.meta = {}
89 self.data = {}
89 self.data = {}
90 self.open_file = h5py.File
90 self.open_file = h5py.File
91 self.open_mode = 'r'
91 self.open_mode = 'r'
92 self.description = {}
92 self.description = {}
93 self.extras = {}
93 self.extras = {}
94 self.filefmt = "*%Y%j***"
94 self.filefmt = "*%Y%j***"
95 self.folderfmt = "*%Y%j"
95 self.folderfmt = "*%Y%j"
96 self.utcoffset = 0
96 self.utcoffset = 0
97
97
98 def setup(self, **kwargs):
98 def setup(self, **kwargs):
99
99
100 self.set_kwargs(**kwargs)
100 self.set_kwargs(**kwargs)
101 if not self.ext.startswith('.'):
101 if not self.ext.startswith('.'):
102 self.ext = '.{}'.format(self.ext)
102 self.ext = '.{}'.format(self.ext)
103
103
104 if self.online:
104 if self.online:
105 log.log("Searching files in online mode...", self.name)
105 log.log("Searching files in online mode...", self.name)
106
106
107 for nTries in range(self.nTries):
107 for nTries in range(self.nTries):
108 fullpath = self.searchFilesOnLine(self.path, self.startDate,
108 fullpath = self.searchFilesOnLine(self.path, self.startDate,
109 self.endDate, self.expLabel, self.ext, self.walk,
109 self.endDate, self.expLabel, self.ext, self.walk,
110 self.filefmt, self.folderfmt)
110 self.filefmt, self.folderfmt)
111 try:
111 try:
112 fullpath = next(fullpath)
112 fullpath = next(fullpath)
113 except:
113 except:
114 fullpath = None
114 fullpath = None
115
115
116 if fullpath:
116 if fullpath:
117 break
117 break
118
118
119 log.warning(
119 log.warning(
120 'Waiting {} sec for a valid file in {}: try {} ...'.format(
120 'Waiting {} sec for a valid file in {}: try {} ...'.format(
121 self.delay, self.path, nTries + 1),
121 self.delay, self.path, nTries + 1),
122 self.name)
122 self.name)
123 time.sleep(self.delay)
123 time.sleep(self.delay)
124
124
125 if not(fullpath):
125 if not(fullpath):
126 raise schainpy.admin.SchainError(
126 raise schainpy.admin.SchainError(
127 'There isn\'t any valid file in {}'.format(self.path))
127 'There isn\'t any valid file in {}'.format(self.path))
128
128
129 pathname, filename = os.path.split(fullpath)
129 pathname, filename = os.path.split(fullpath)
130 self.year = int(filename[1:5])
130 self.year = int(filename[1:5])
131 self.doy = int(filename[5:8])
131 self.doy = int(filename[5:8])
132 self.set = int(filename[8:11]) - 1
132 self.set = int(filename[8:11]) - 1
133 else:
133 else:
134 log.log("Searching files in {}".format(self.path), self.name)
134 log.log("Searching files in {}".format(self.path), self.name)
135 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
135 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
136 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
136 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
137
137
138 self.setNextFile()
138 self.setNextFile()
139
139
140 return
140 return
141
141
142 def readFirstHeader(self):
142 def readFirstHeader(self):
143 '''Read metadata and data'''
143 '''Read metadata and data'''
144
144
145 self.__readMetadata()
145 self.__readMetadata()
146 self.__readData()
146 self.__readData()
147 self.__setBlockList()
147 self.__setBlockList()
148
148 # similar to master
149 if 'type' in self.meta:
150 self.dataOut = eval(self.meta['type'])()
151 # similar to master
149 for attr in self.meta:
152 for attr in self.meta:
150 setattr(self.dataOut, attr, self.meta[attr])
153 setattr(self.dataOut, attr, self.meta[attr])
151
154
152 self.blockIndex = 0
155 self.blockIndex = 0
153
156
154 return
157 return
155
158
156 def __setBlockList(self):
159 def __setBlockList(self):
157 '''
160 '''
158 Selects the data within the times defined
161 Selects the data within the times defined
159
162
160 self.fp
163 self.fp
161 self.startTime
164 self.startTime
162 self.endTime
165 self.endTime
163 self.blockList
166 self.blockList
164 self.blocksPerFile
167 self.blocksPerFile
165
168
166 '''
169 '''
167
170
168 startTime = self.startTime
171 startTime = self.startTime
169 endTime = self.endTime
172 endTime = self.endTime
170 thisUtcTime = self.data['utctime'] + self.utcoffset
173 thisUtcTime = self.data['utctime'] + self.utcoffset
171 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
174 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
172 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
175 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
173
176
174 thisDate = thisDatetime.date()
177 thisDate = thisDatetime.date()
175 thisTime = thisDatetime.time()
178 thisTime = thisDatetime.time()
176
179
177 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
180 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
178 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
181 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
179
182
180 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
183 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
181
184
182 self.blockList = ind
185 self.blockList = ind
183 self.blocksPerFile = len(ind)
186 self.blocksPerFile = len(ind)
187 # similar to master
188 if len(ind)==0:
189 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.blockIndex,
190 self.blocksPerFile,
191 thisDatetime))
192 self.setNextFile()
193 # similar to master
184 return
194 return
185
195
186 def __readMetadata(self):
196 def __readMetadata(self):
187 '''
197 '''
188 Reads Metadata
198 Reads Metadata
189 '''
199 '''
190
200
191 meta = {}
201 meta = {}
192
202
193 if self.description:
203 if self.description:
194 for key, value in self.description['Metadata'].items():
204 for key, value in self.description['Metadata'].items():
195 meta[key] = self.fp[value][()]
205 meta[key] = self.fp[value][()]
196 else:
206 else:
197 grp = self.fp['Metadata']
207 grp = self.fp['Metadata']
198 for name in grp:
208 for name in grp:
199 meta[name] = grp[name][()]
209 meta[name] = grp[name][()]
200
210
201 if self.extras:
211 if self.extras:
202 for key, value in self.extras.items():
212 for key, value in self.extras.items():
203 meta[key] = value
213 meta[key] = value
204 self.meta = meta
214 self.meta = meta
205
215
206 return
216 return
207
217
208 def __readData(self):
218 def __readData(self):
209
219
210 data = {}
220 data = {}
211
221
212 if self.description:
222 if self.description:
213 for key, value in self.description['Data'].items():
223 for key, value in self.description['Data'].items():
214 if isinstance(value, str):
224 if isinstance(value, str):
215 if isinstance(self.fp[value], h5py.Dataset):
225 if isinstance(self.fp[value], h5py.Dataset):
216 data[key] = self.fp[value][()]
226 data[key] = self.fp[value][()]
217 elif isinstance(self.fp[value], h5py.Group):
227 elif isinstance(self.fp[value], h5py.Group):
218 array = []
228 array = []
219 for ch in self.fp[value]:
229 for ch in self.fp[value]:
220 array.append(self.fp[value][ch][()])
230 array.append(self.fp[value][ch][()])
221 data[key] = numpy.array(array)
231 data[key] = numpy.array(array)
222 elif isinstance(value, list):
232 elif isinstance(value, list):
223 array = []
233 array = []
224 for ch in value:
234 for ch in value:
225 array.append(self.fp[ch][()])
235 array.append(self.fp[ch][()])
226 data[key] = numpy.array(array)
236 data[key] = numpy.array(array)
227 else:
237 else:
228 grp = self.fp['Data']
238 grp = self.fp['Data']
229 for name in grp:
239 for name in grp:
230 if isinstance(grp[name], h5py.Dataset):
240 if isinstance(grp[name], h5py.Dataset):
231 array = grp[name][()]
241 array = grp[name][()]
232 elif isinstance(grp[name], h5py.Group):
242 elif isinstance(grp[name], h5py.Group):
233 array = []
243 array = []
234 for ch in grp[name]:
244 for ch in grp[name]:
235 array.append(grp[name][ch][()])
245 array.append(grp[name][ch][()])
236 array = numpy.array(array)
246 array = numpy.array(array)
237 else:
247 else:
238 log.warning('Unknown type: {}'.format(name))
248 log.warning('Unknown type: {}'.format(name))
239
249
240 if name in self.description:
250 if name in self.description:
241 key = self.description[name]
251 key = self.description[name]
242 else:
252 else:
243 key = name
253 key = name
244 data[key] = array
254 data[key] = array
245
255
246 self.data = data
256 self.data = data
247 return
257 return
248
258
249 def getData(self):
259 def getData(self):
250
260
251 for attr in self.data:
261 for attr in self.data:
252 if self.data[attr].ndim == 1:
262 if self.data[attr].ndim == 1:
253 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
263 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
254 else:
264 else:
255 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
265 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
256
266
257 self.dataOut.flagNoData = False
267 self.dataOut.flagNoData = False
258 self.blockIndex += 1
268 self.blockIndex += 1
259
269
260 log.log("Block No. {}/{} -> {}".format(
270 log.log("Block No. {}/{} -> {}".format(
261 self.blockIndex,
271 self.blockIndex,
262 self.blocksPerFile,
272 self.blocksPerFile,
263 self.dataOut.datatime.ctime()), self.name)
273 self.dataOut.datatime.ctime()), self.name)
264
274
265 return
275 return
266
276
267 def run(self, **kwargs):
277 def run(self, **kwargs):
268
278
269 if not(self.isConfig):
279 if not(self.isConfig):
270 self.setup(**kwargs)
280 self.setup(**kwargs)
271 self.isConfig = True
281 self.isConfig = True
272
282
273 if self.blockIndex == self.blocksPerFile:
283 if self.blockIndex == self.blocksPerFile:
274 self.setNextFile()
284 self.setNextFile()
275
285
276 self.getData()
286 self.getData()
277
287 ''' # this block is missing in master.
278 if 'type' in self.meta:
288 if 'type' in self.meta:
279 self.dataOut.type = self.meta['type'].decode('utf-8')
289 self.dataOut.type = self.meta['type'].decode('utf-8')
280
290 '''
281 return
291 return
282
292
283 @MPDecorator
293 @MPDecorator
284 class HDFWriter(Operation):
294 class HDFWriter(Operation):
285 """Operation to write HDF5 files.
295 """Operation to write HDF5 files.
286
296
287 The HDF5 file contains by default two groups Data and Metadata where
297 The HDF5 file contains by default two groups Data and Metadata where
288 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
298 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
289 parameters, data attributes are normaly time dependent where the metadata
299 parameters, data attributes are normaly time dependent where the metadata
290 are not.
300 are not.
291 It is possible to customize the structure of the HDF5 file with the
301 It is possible to customize the structure of the HDF5 file with the
292 optional description parameter see the examples.
302 optional description parameter see the examples.
293
303
294 Parameters:
304 Parameters:
295 -----------
305 -----------
296 path : str
306 path : str
297 Path where files will be saved.
307 Path where files will be saved.
298 blocksPerFile : int
308 blocksPerFile : int
299 Number of blocks per file
309 Number of blocks per file
300 metadataList : list
310 metadataList : list
301 List of the dataOut attributes that will be saved as metadata
311 List of the dataOut attributes that will be saved as metadata
302 dataList : int
312 dataList : int
303 List of the dataOut attributes that will be saved as data
313 List of the dataOut attributes that will be saved as data
304 setType : bool
314 setType : bool
305 If True the name of the files corresponds to the timestamp of the data
315 If True the name of the files corresponds to the timestamp of the data
306 description : dict, optional
316 description : dict, optional
307 Dictionary with the desired description of the HDF5 file
317 Dictionary with the desired description of the HDF5 file
308
318
309 Examples
319 Examples
310 --------
320 --------
311
321
312 desc = {
322 desc = {
313 'data_output': {'winds': ['z', 'w', 'v']},
323 'data_output': {'winds': ['z', 'w', 'v']},
314 'utctime': 'timestamps',
324 'utctime': 'timestamps',
315 'heightList': 'heights'
325 'heightList': 'heights'
316 }
326 }
317 desc = {
327 desc = {
318 'data_output': ['z', 'w', 'v'],
328 'data_output': ['z', 'w', 'v'],
319 'utctime': 'timestamps',
329 'utctime': 'timestamps',
320 'heightList': 'heights'
330 'heightList': 'heights'
321 }
331 }
322 desc = {
332 desc = {
323 'Data': {
333 'Data': {
324 'data_output': 'winds',
334 'data_output': 'winds',
325 'utctime': 'timestamps'
335 'utctime': 'timestamps'
326 },
336 },
327 'Metadata': {
337 'Metadata': {
328 'heightList': 'heights'
338 'heightList': 'heights'
329 }
339 }
330 }
340 }
331
341
332 writer = proc_unit.addOperation(name='HDFWriter')
342 writer = proc_unit.addOperation(name='HDFWriter')
333 writer.addParameter(name='path', value='/path/to/file')
343 writer.addParameter(name='path', value='/path/to/file')
334 writer.addParameter(name='blocksPerFile', value='32')
344 writer.addParameter(name='blocksPerFile', value='32')
335 writer.addParameter(name='metadataList', value='heightList,timeZone')
345 writer.addParameter(name='metadataList', value='heightList,timeZone')
336 writer.addParameter(name='dataList',value='data_output,utctime')
346 writer.addParameter(name='dataList',value='data_output,utctime')
337 # writer.addParameter(name='description',value=json.dumps(desc))
347 # writer.addParameter(name='description',value=json.dumps(desc))
338
348
339 """
349 """
340
350
341 ext = ".hdf5"
351 ext = ".hdf5"
342 optchar = "D"
352 optchar = "D"
343 filename = None
353 filename = None
344 path = None
354 path = None
345 setFile = None
355 setFile = None
346 fp = None
356 fp = None
347 firsttime = True
357 firsttime = True
348 # Configurations
358 # Configurations
349 blocksPerFile = None
359 blocksPerFile = None
350 blockIndex = None
360 blockIndex = None
351 dataOut = None
361 dataOut = None
352 # Data Arrays
362 # Data Arrays
353 dataList = None
363 dataList = None
354 metadataList = None
364 metadataList = None
355 currentDay = None
365 currentDay = None
356 lastTime = None
366 lastTime = None
357
367
358 def __init__(self):
368 def __init__(self):
359
369
360 Operation.__init__(self)
370 Operation.__init__(self)
361 return
371 return
362
372
363 def set_kwargs(self, **kwargs):
373 def set_kwargs(self, **kwargs):
364
374
365 for key, value in kwargs.items():
375 for key, value in kwargs.items():
366 setattr(self, key, value)
376 setattr(self, key, value)
367
377
368 def set_kwargs_obj(self, obj, **kwargs):
378 def set_kwargs_obj(self, obj, **kwargs):
369
379
370 for key, value in kwargs.items():
380 for key, value in kwargs.items():
371 setattr(obj, key, value)
381 setattr(obj, key, value)
372
382
373 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None, **kwargs):
383 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None, **kwargs):
374 self.path = path
384 self.path = path
375 self.blocksPerFile = blocksPerFile
385 self.blocksPerFile = blocksPerFile
376 self.metadataList = metadataList
386 self.metadataList = metadataList
377 self.dataList = [s.strip() for s in dataList]
387 self.dataList = [s.strip() for s in dataList]
378 self.setType = setType
388 self.setType = setType
379 self.description = description
389 self.description = description
380 self.set_kwargs(**kwargs)
390 self.set_kwargs(**kwargs)
381
391
382 if self.metadataList is None:
392 if self.metadataList is None:
383 self.metadataList = self.dataOut.metadata_list
393 self.metadataList = self.dataOut.metadata_list
384
394
385 tableList = []
395 tableList = []
386 dsList = []
396 dsList = []
387
397
388 for i in range(len(self.dataList)):
398 for i in range(len(self.dataList)):
389 dsDict = {}
399 dsDict = {}
390 if hasattr(self.dataOut, self.dataList[i]):
400 if hasattr(self.dataOut, self.dataList[i]):
391 dataAux = getattr(self.dataOut, self.dataList[i])
401 dataAux = getattr(self.dataOut, self.dataList[i])
392 dsDict['variable'] = self.dataList[i]
402 dsDict['variable'] = self.dataList[i]
393 else:
403 else:
394 log.warning('Attribute {} not found in dataOut', self.name)
404 log.warning('Attribute {} not found in dataOut', self.name)
395 continue
405 continue
396
406
397 if dataAux is None:
407 if dataAux is None:
398 continue
408 continue
399 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float32)):
409 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float32)):
400 dsDict['nDim'] = 0
410 dsDict['nDim'] = 0
401 else:
411 else:
402 dsDict['nDim'] = len(dataAux.shape)
412 dsDict['nDim'] = len(dataAux.shape)
403 dsDict['shape'] = dataAux.shape
413 dsDict['shape'] = dataAux.shape
404 dsDict['dsNumber'] = dataAux.shape[0]
414 dsDict['dsNumber'] = dataAux.shape[0]
405 dsDict['dtype'] = dataAux.dtype
415 dsDict['dtype'] = dataAux.dtype
406
416
407 dsList.append(dsDict)
417 dsList.append(dsDict)
408
418
409 self.dsList = dsList
419 self.dsList = dsList
410 self.currentDay = self.dataOut.datatime.date()
420 self.currentDay = self.dataOut.datatime.date()
411
421
412 def timeFlag(self):
422 def timeFlag(self):
413 currentTime = self.dataOut.utctime
423 currentTime = self.dataOut.utctime
414 timeTuple = time.localtime(currentTime)
424 timeTuple = time.localtime(currentTime)
415 dataDay = timeTuple.tm_yday
425 dataDay = timeTuple.tm_yday
416
426
417 if self.lastTime is None:
427 if self.lastTime is None:
418 self.lastTime = currentTime
428 self.lastTime = currentTime
419 self.currentDay = dataDay
429 self.currentDay = dataDay
420 return False
430 return False
421
431
422 timeDiff = currentTime - self.lastTime
432 timeDiff = currentTime - self.lastTime
423
433
424 # Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
434 # Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
425 if dataDay != self.currentDay:
435 if dataDay != self.currentDay:
426 self.currentDay = dataDay
436 self.currentDay = dataDay
427 return True
437 return True
428 elif timeDiff > 3 * 60 * 60:
438 elif timeDiff > 3 * 60 * 60:
429 self.lastTime = currentTime
439 self.lastTime = currentTime
430 return True
440 return True
431 else:
441 else:
432 self.lastTime = currentTime
442 self.lastTime = currentTime
433 return False
443 return False
434
444
435 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
445 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
436 dataList=[], setType=None, description={}, **kwargs):
446 dataList=[], setType=None, description={}, **kwargs):
437
447
438 self.dataOut = dataOut
448 self.dataOut = dataOut
439 self.set_kwargs_obj(self.dataOut, **kwargs)
449 self.set_kwargs_obj(self.dataOut, **kwargs)
440 if not(self.isConfig):
450 if not(self.isConfig):
441 self.setup(path=path, blocksPerFile=blocksPerFile,
451 self.setup(path=path, blocksPerFile=blocksPerFile,
442 metadataList=metadataList, dataList=dataList,
452 metadataList=metadataList, dataList=dataList,
443 setType=setType, description=description, **kwargs)
453 setType=setType, description=description, **kwargs)
444
454
445 self.isConfig = True
455 self.isConfig = True
446 self.setNextFile()
456 self.setNextFile()
447
457
448 self.putData()
458 self.putData()
449 return
459 return
450
460
451 def setNextFile(self):
461 def setNextFile(self):
452
462
453 ext = self.ext
463 ext = self.ext
454 path = self.path
464 path = self.path
455 setFile = self.setFile
465 setFile = self.setFile
456
466
457 timeTuple = time.localtime(self.dataOut.utctime)
467 timeTuple = time.localtime(self.dataOut.utctime)
458 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
468 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
459 fullpath = os.path.join(path, subfolder)
469 fullpath = os.path.join(path, subfolder)
460
470
461 if os.path.exists(fullpath):
471 if os.path.exists(fullpath):
462 filesList = os.listdir(fullpath)
472 filesList = os.listdir(fullpath)
463 filesList = [k for k in filesList if k.startswith(self.optchar)]
473 filesList = [k for k in filesList if k.startswith(self.optchar)]
464 if len(filesList) > 0:
474 if len(filesList) > 0:
465 filesList = sorted(filesList, key=str.lower)
475 filesList = sorted(filesList, key=str.lower)
466 filen = filesList[-1]
476 filen = filesList[-1]
467 # el filename debera tener el siguiente formato
477 # el filename debera tener el siguiente formato
468 # 0 1234 567 89A BCDE (hex)
478 # 0 1234 567 89A BCDE (hex)
469 # x YYYY DDD SSS .ext
479 # x YYYY DDD SSS .ext
470 if isNumber(filen[8:11]):
480 if isNumber(filen[8:11]):
471 setFile = int(filen[8:11]) # inicializo mi contador de seteo al seteo del ultimo file
481 setFile = int(filen[8:11]) # inicializo mi contador de seteo al seteo del ultimo file
472 else:
482 else:
473 setFile = -1
483 setFile = -1
474 else:
484 else:
475 setFile = -1 # inicializo mi contador de seteo
485 setFile = -1 # inicializo mi contador de seteo
476 else:
486 else:
477 os.makedirs(fullpath)
487 os.makedirs(fullpath)
478 setFile = -1 # inicializo mi contador de seteo
488 setFile = -1 # inicializo mi contador de seteo
479
489
480 if self.setType is None:
490 if self.setType is None:
481 setFile += 1
491 setFile += 1
482 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
492 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
483 timeTuple.tm_year,
493 timeTuple.tm_year,
484 timeTuple.tm_yday,
494 timeTuple.tm_yday,
485 setFile,
495 setFile,
486 ext)
496 ext)
487 else:
497 else:
488 setFile = timeTuple.tm_hour * 60 + timeTuple.tm_min
498 setFile = timeTuple.tm_hour * 60 + timeTuple.tm_min
489 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
499 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
490 timeTuple.tm_year,
500 timeTuple.tm_year,
491 timeTuple.tm_yday,
501 timeTuple.tm_yday,
492 setFile,
502 setFile,
493 ext)
503 ext)
494
504
495 self.filename = os.path.join(path, subfolder, file)
505 self.filename = os.path.join(path, subfolder, file)
496
506
497 # Setting HDF5 File
507 # Setting HDF5 File
498 self.fp = h5py.File(self.filename, 'w')
508 self.fp = h5py.File(self.filename, 'w')
499 # write metadata
509 # write metadata
500 self.writeMetadata(self.fp)
510 self.writeMetadata(self.fp)
501 # Write data
511 # Write data
502 self.writeData(self.fp)
512 self.writeData(self.fp)
503
513
504 def getLabel(self, name, x=None):
514 def getLabel(self, name, x=None):
505
515
506 if x is None:
516 if x is None:
507 if 'Data' in self.description:
517 if 'Data' in self.description:
508 data = self.description['Data']
518 data = self.description['Data']
509 if 'Metadata' in self.description:
519 if 'Metadata' in self.description:
510 data.update(self.description['Metadata'])
520 data.update(self.description['Metadata'])
511 else:
521 else:
512 data = self.description
522 data = self.description
513 if name in data:
523 if name in data:
514 if isinstance(data[name], str):
524 if isinstance(data[name], str):
515 return data[name]
525 return data[name]
516 elif isinstance(data[name], list):
526 elif isinstance(data[name], list):
517 return None
527 return None
518 elif isinstance(data[name], dict):
528 elif isinstance(data[name], dict):
519 for key, value in data[name].items():
529 for key, value in data[name].items():
520 return key
530 return key
521 return name
531 return name
522 else:
532 else:
523 if 'Data' in self.description:
533 if 'Data' in self.description:
524 data = self.description['Data']
534 data = self.description['Data']
525 if 'Metadata' in self.description:
535 if 'Metadata' in self.description:
526 data.update(self.description['Metadata'])
536 data.update(self.description['Metadata'])
527 else:
537 else:
528 data = self.description
538 data = self.description
529 if name in data:
539 if name in data:
530 if isinstance(data[name], list):
540 if isinstance(data[name], list):
531 return data[name][x]
541 return data[name][x]
532 elif isinstance(data[name], dict):
542 elif isinstance(data[name], dict):
533 for key, value in data[name].items():
543 for key, value in data[name].items():
534 return value[x]
544 return value[x]
535 if 'cspc' in name:
545 if 'cspc' in name:
536 return 'pair{:02d}'.format(x)
546 return 'pair{:02d}'.format(x)
537 else:
547 else:
538 return 'channel{:02d}'.format(x)
548 return 'channel{:02d}'.format(x)
539
549
540 def writeMetadata(self, fp):
550 def writeMetadata(self, fp):
541
551
542 if self.description:
552 if self.description:
543 if 'Metadata' in self.description:
553 if 'Metadata' in self.description:
544 grp = fp.create_group('Metadata')
554 grp = fp.create_group('Metadata')
545 else:
555 else:
546 grp = fp
556 grp = fp
547 else:
557 else:
548 grp = fp.create_group('Metadata')
558 grp = fp.create_group('Metadata')
549
559
550 for i in range(len(self.metadataList)):
560 for i in range(len(self.metadataList)):
551 if not hasattr(self.dataOut, self.metadataList[i]):
561 if not hasattr(self.dataOut, self.metadataList[i]):
552 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
562 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
553 continue
563 continue
554 value = getattr(self.dataOut, self.metadataList[i])
564 value = getattr(self.dataOut, self.metadataList[i])
555 if isinstance(value, bool):
565 if isinstance(value, bool):
556 if value is True:
566 if value is True:
557 value = 1
567 value = 1
558 else:
568 else:
559 value = 0
569 value = 0
560 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
570 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
561 return
571 return
562
572
563 def writeData(self, fp):
573 def writeData(self, fp):
564
574
565 if self.description:
575 if self.description:
566 if 'Data' in self.description:
576 if 'Data' in self.description:
567 grp = fp.create_group('Data')
577 grp = fp.create_group('Data')
568 else:
578 else:
569 grp = fp
579 grp = fp
570 else:
580 else:
571 grp = fp.create_group('Data')
581 grp = fp.create_group('Data')
572
582
573 dtsets = []
583 dtsets = []
574 data = []
584 data = []
575
585
576 for dsInfo in self.dsList:
586 for dsInfo in self.dsList:
577 if dsInfo['nDim'] == 0:
587 if dsInfo['nDim'] == 0:
578 ds = grp.create_dataset(
588 ds = grp.create_dataset(
579 self.getLabel(dsInfo['variable']),
589 self.getLabel(dsInfo['variable']),
580 (self.blocksPerFile,),
590 (self.blocksPerFile,),
581 chunks=True,
591 chunks=True,
582 dtype=numpy.float64)
592 dtype=numpy.float64)
583 dtsets.append(ds)
593 dtsets.append(ds)
584 data.append((dsInfo['variable'], -1))
594 data.append((dsInfo['variable'], -1))
585 else:
595 else:
586 label = self.getLabel(dsInfo['variable'])
596 label = self.getLabel(dsInfo['variable'])
587 if label is not None:
597 if label is not None:
588 sgrp = grp.create_group(label)
598 sgrp = grp.create_group(label)
589 else:
599 else:
590 sgrp = grp
600 sgrp = grp
591 for i in range(dsInfo['dsNumber']):
601 for i in range(dsInfo['dsNumber']):
592 ds = sgrp.create_dataset(
602 ds = sgrp.create_dataset(
593 self.getLabel(dsInfo['variable'], i),
603 self.getLabel(dsInfo['variable'], i),
594 (self.blocksPerFile,) + dsInfo['shape'][1:],
604 (self.blocksPerFile,) + dsInfo['shape'][1:],
595 chunks=True,
605 chunks=True,
596 dtype=dsInfo['dtype'])
606 dtype=dsInfo['dtype'])
597 dtsets.append(ds)
607 dtsets.append(ds)
598 data.append((dsInfo['variable'], i))
608 data.append((dsInfo['variable'], i))
599 fp.flush()
609 fp.flush()
600
610
601 log.log('Creating file: {}'.format(fp.filename), self.name)
611 log.log('Creating file: {}'.format(fp.filename), self.name)
602
612
603 self.ds = dtsets
613 self.ds = dtsets
604 self.data = data
614 self.data = data
605 self.firsttime = True
615 self.firsttime = True
606 self.blockIndex = 0
616 self.blockIndex = 0
607 return
617 return
608
618
609 def putData(self):
619 def putData(self):
610
620
611 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
621 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
612 self.closeFile()
622 self.closeFile()
613 self.setNextFile()
623 self.setNextFile()
614
624
615 for i, ds in enumerate(self.ds):
625 for i, ds in enumerate(self.ds):
616 attr, ch = self.data[i]
626 attr, ch = self.data[i]
617 if ch == -1:
627 if ch == -1:
618 ds[self.blockIndex] = getattr(self.dataOut, attr)
628 ds[self.blockIndex] = getattr(self.dataOut, attr)
619 else:
629 else:
620 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
630 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
621
631
622 self.fp.flush()
632 self.fp.flush()
623 self.blockIndex += 1
633 self.blockIndex += 1
624 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
634 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
625
635
626 return
636 return
627
637
628 def closeFile(self):
638 def closeFile(self):
629
639
630 if self.blockIndex != self.blocksPerFile:
640 if self.blockIndex != self.blocksPerFile:
631 for ds in self.ds:
641 for ds in self.ds:
632 ds.resize(self.blockIndex, axis=0)
642 ds.resize(self.blockIndex, axis=0)
633
643
634 if self.fp:
644 if self.fp:
635 self.fp.flush()
645 self.fp.flush()
636 self.fp.close()
646 self.fp.close()
637
647
638 def close(self):
648 def close(self):
639
649
640 self.closeFile()
650 self.closeFile()
641
651
642
652
643 @MPDecorator
653 @MPDecorator
644 class ASCIIWriter(Operation):
654 class ASCIIWriter(Operation):
645 """Operation to write data in ascii files.
655 """Operation to write data in ascii files.
646
656
647 Parameters:
657 Parameters:
648 -----------
658 -----------
649 path : str
659 path : str
650 Path where files will be saved.
660 Path where files will be saved.
651 blocksPerFile : int
661 blocksPerFile : int
652 Number of blocks per file
662 Number of blocks per file
653 metadataList : list
663 metadataList : list
654 List of the dataOut attributes that will be saved as metadata
664 List of the dataOut attributes that will be saved as metadata
655 dataDict : dict
665 dataDict : dict
656 Dictionary with the varaibles to be saved
666 Dictionary with the varaibles to be saved
657 setType : bool
667 setType : bool
658 If True the name of the files corresponds to the timestamp of the data
668 If True the name of the files corresponds to the timestamp of the data
659
669
660 Examples
670 Examples
661 --------
671 --------
662
672
663 data = {
673 data = {
664 'data_output': ['z', 'w', 'v'],
674 'data_output': ['z', 'w', 'v'],
665 'utctime': 'time',
675 'utctime': 'time',
666 'heightList': 'height'
676 'heightList': 'height'
667 }
677 }
668
678
669 writer = proc_unit.addOperation(name='ASCIIWriter')
679 writer = proc_unit.addOperation(name='ASCIIWriter')
670 writer.addParameter(name='path', value='/path/to/file')
680 writer.addParameter(name='path', value='/path/to/file')
671 writer.addParameter(name='blocksPerFile', value='32')
681 writer.addParameter(name='blocksPerFile', value='32')
672 writer.addParameter(name='dataDict',value=json.dumps(data))
682 writer.addParameter(name='dataDict',value=json.dumps(data))
673
683
674 """
684 """
675
685
676 ext = ".txt"
686 ext = ".txt"
677 optchar = "D"
687 optchar = "D"
678 filename = None
688 filename = None
679 path = None
689 path = None
680 setFile = None
690 setFile = None
681 fp = None
691 fp = None
682 firsttime = True
692 firsttime = True
683 # Configurations
693 # Configurations
684 blocksPerFile = None
694 blocksPerFile = None
685 blockIndex = None
695 blockIndex = None
686 dataOut = None
696 dataOut = None
687 # Data Arrays
697 # Data Arrays
688 dataDict = None
698 dataDict = None
689 metadataList = None
699 metadataList = None
690 currentDay = None
700 currentDay = None
691 lastTime = None
701 lastTime = None
692 localtime = True
702 localtime = True
693
703
694 def __init__(self):
704 def __init__(self):
695
705
696 Operation.__init__(self)
706 Operation.__init__(self)
697 return
707 return
698
708
699 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataDict=None, setType=None, localtime=True):
709 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataDict=None, setType=None, localtime=True):
700 self.path = path
710 self.path = path
701 self.blocksPerFile = blocksPerFile
711 self.blocksPerFile = blocksPerFile
702 self.metadataList = metadataList
712 self.metadataList = metadataList
703 self.dataDict = dataDict
713 self.dataDict = dataDict
704 self.setType = setType
714 self.setType = setType
705 self.localtime = localtime
715 self.localtime = localtime
706
716
707 if self.metadataList is None:
717 if self.metadataList is None:
708 self.metadataList = self.dataOut.metadata_list
718 self.metadataList = self.dataOut.metadata_list
709
719
710 dsList = []
720 dsList = []
711
721
712 for key, value in self.dataDict.items():
722 for key, value in self.dataDict.items():
713 dsDict = {}
723 dsDict = {}
714 if hasattr(self.dataOut, key):
724 if hasattr(self.dataOut, key):
715 dataAux = getattr(self.dataOut, key)
725 dataAux = getattr(self.dataOut, key)
716 dsDict['variable'] = key
726 dsDict['variable'] = key
717 else:
727 else:
718 log.warning('Attribute {} not found in dataOut', self.name)
728 log.warning('Attribute {} not found in dataOut', self.name)
719 continue
729 continue
720
730
721 if dataAux is None:
731 if dataAux is None:
722 continue
732 continue
723 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float32)):
733 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float32)):
724 dsDict['nDim'] = 0
734 dsDict['nDim'] = 0
725 else:
735 else:
726 dsDict['nDim'] = len(dataAux.shape)
736 dsDict['nDim'] = len(dataAux.shape)
727 dsDict['shape'] = dataAux.shape
737 dsDict['shape'] = dataAux.shape
728 dsDict['dsNumber'] = dataAux.shape[0]
738 dsDict['dsNumber'] = dataAux.shape[0]
729 dsDict['dtype'] = dataAux.dtype
739 dsDict['dtype'] = dataAux.dtype
730
740
731 dsList.append(dsDict)
741 dsList.append(dsDict)
732 self.dsList = dsList
742 self.dsList = dsList
733 self.currentDay = self.dataOut.datatime.date()
743 self.currentDay = self.dataOut.datatime.date()
734
744
735 def timeFlag(self):
745 def timeFlag(self):
736 currentTime = self.dataOut.utctime
746 currentTime = self.dataOut.utctime
737 if self.localtime:
747 if self.localtime:
738 timeTuple = time.localtime(currentTime)
748 timeTuple = time.localtime(currentTime)
739 else:
749 else:
740 timeTuple = time.gmtime(currentTime)
750 timeTuple = time.gmtime(currentTime)
741
751
742 dataDay = timeTuple.tm_yday
752 dataDay = timeTuple.tm_yday
743
753
744 if self.lastTime is None:
754 if self.lastTime is None:
745 self.lastTime = currentTime
755 self.lastTime = currentTime
746 self.currentDay = dataDay
756 self.currentDay = dataDay
747 return False
757 return False
748
758
749 timeDiff = currentTime - self.lastTime
759 timeDiff = currentTime - self.lastTime
750
760
751 # Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
761 # Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
752 if dataDay != self.currentDay:
762 if dataDay != self.currentDay:
753 self.currentDay = dataDay
763 self.currentDay = dataDay
754 return True
764 return True
755 elif timeDiff > 3 * 60 * 60:
765 elif timeDiff > 3 * 60 * 60:
756 self.lastTime = currentTime
766 self.lastTime = currentTime
757 return True
767 return True
758 else:
768 else:
759 self.lastTime = currentTime
769 self.lastTime = currentTime
760 return False
770 return False
761
771
762 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
772 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
763 dataDict={}, setType=None, localtime=True):
773 dataDict={}, setType=None, localtime=True):
764
774
765 self.dataOut = dataOut
775 self.dataOut = dataOut
766 if not(self.isConfig):
776 if not(self.isConfig):
767 self.setup(path=path, blocksPerFile=blocksPerFile,
777 self.setup(path=path, blocksPerFile=blocksPerFile,
768 metadataList=metadataList, dataDict=dataDict,
778 metadataList=metadataList, dataDict=dataDict,
769 setType=setType, localtime=localtime)
779 setType=setType, localtime=localtime)
770
780
771 self.isConfig = True
781 self.isConfig = True
772 self.setNextFile()
782 self.setNextFile()
773
783
774 self.putData()
784 self.putData()
775 return
785 return
776
786
777 def setNextFile(self):
787 def setNextFile(self):
778
788
779 ext = self.ext
789 ext = self.ext
780 path = self.path
790 path = self.path
781 setFile = self.setFile
791 setFile = self.setFile
782 if self.localtime:
792 if self.localtime:
783 timeTuple = time.localtime(self.dataOut.utctime)
793 timeTuple = time.localtime(self.dataOut.utctime)
784 else:
794 else:
785 timeTuple = time.gmtime(self.dataOut.utctime)
795 timeTuple = time.gmtime(self.dataOut.utctime)
786 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
796 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
787 fullpath = os.path.join(path, subfolder)
797 fullpath = os.path.join(path, subfolder)
788
798
789 if os.path.exists(fullpath):
799 if os.path.exists(fullpath):
790 filesList = os.listdir(fullpath)
800 filesList = os.listdir(fullpath)
791 filesList = [k for k in filesList if k.startswith(self.optchar)]
801 filesList = [k for k in filesList if k.startswith(self.optchar)]
792 if len(filesList) > 0:
802 if len(filesList) > 0:
793 filesList = sorted(filesList, key=str.lower)
803 filesList = sorted(filesList, key=str.lower)
794 filen = filesList[-1]
804 filen = filesList[-1]
795 # el filename debera tener el siguiente formato
805 # el filename debera tener el siguiente formato
796 # 0 1234 567 89A BCDE (hex)
806 # 0 1234 567 89A BCDE (hex)
797 # x YYYY DDD SSS .ext
807 # x YYYY DDD SSS .ext
798 if isNumber(filen[8:11]):
808 if isNumber(filen[8:11]):
799 setFile = int(filen[8:11]) # inicializo mi contador de seteo al seteo del ultimo file
809 setFile = int(filen[8:11]) # inicializo mi contador de seteo al seteo del ultimo file
800 else:
810 else:
801 setFile = -1
811 setFile = -1
802 else:
812 else:
803 setFile = -1 # inicializo mi contador de seteo
813 setFile = -1 # inicializo mi contador de seteo
804 else:
814 else:
805 os.makedirs(fullpath)
815 os.makedirs(fullpath)
806 setFile = -1 # inicializo mi contador de seteo
816 setFile = -1 # inicializo mi contador de seteo
807
817
808 if self.setType is None:
818 if self.setType is None:
809 setFile += 1
819 setFile += 1
810 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
820 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
811 timeTuple.tm_year,
821 timeTuple.tm_year,
812 timeTuple.tm_yday,
822 timeTuple.tm_yday,
813 setFile,
823 setFile,
814 ext)
824 ext)
815 else:
825 else:
816 setFile = timeTuple.tm_hour * 60 + timeTuple.tm_min
826 setFile = timeTuple.tm_hour * 60 + timeTuple.tm_min
817 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
827 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
818 timeTuple.tm_year,
828 timeTuple.tm_year,
819 timeTuple.tm_yday,
829 timeTuple.tm_yday,
820 setFile,
830 setFile,
821 ext)
831 ext)
822
832
823 self.filename = os.path.join(path, subfolder, file)
833 self.filename = os.path.join(path, subfolder, file)
824
834
825 # Setting HDF5 File
835 # Setting HDF5 File
826 self.fp = open(self.filename, 'w')
836 self.fp = open(self.filename, 'w')
827 # write metadata
837 # write metadata
828 self.writeMetadata(self.fp)
838 self.writeMetadata(self.fp)
829 # Write data
839 # Write data
830 self.writeData(self.fp)
840 self.writeData(self.fp)
831
841
832 def writeMetadata(self, fp):
842 def writeMetadata(self, fp):
833
843
834 line = ''
844 line = ''
835 for d in self.dsList:
845 for d in self.dsList:
836 par = self.dataDict[d['variable']]
846 par = self.dataDict[d['variable']]
837 if isinstance(par, (list,tuple)):
847 if isinstance(par, (list,tuple)):
838 for p in par:
848 for p in par:
839 line += '{:>16}'.format(p)
849 line += '{:>16}'.format(p)
840 else:
850 else:
841 line += '{:>16}'.format(par)
851 line += '{:>16}'.format(par)
842
852
843 line += '\n'
853 line += '\n'
844 fp.write(line)
854 fp.write(line)
845
855
846 def writeData(self, fp):
856 def writeData(self, fp):
847
857
848 log.log('Creating file: {}'.format(self.filename), self.name)
858 log.log('Creating file: {}'.format(self.filename), self.name)
849
859
850 self.firsttime = True
860 self.firsttime = True
851 self.blockIndex = 0
861 self.blockIndex = 0
852 return
862 return
853
863
854 def putData(self):
864 def putData(self):
855
865
856 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
866 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
857 self.closeFile()
867 self.closeFile()
858 self.setNextFile()
868 self.setNextFile()
859
869
860 line = ''
870 line = ''
861 for j in range(len(self.dataOut.heightList)):
871 for j in range(len(self.dataOut.heightList)):
862 for ds in self.dsList:
872 for ds in self.dsList:
863 par = self.dataDict[ds['variable']]
873 par = self.dataDict[ds['variable']]
864 if ds['nDim'] == 2:
874 if ds['nDim'] == 2:
865 for i in range(len(par)):
875 for i in range(len(par)):
866 line += '{:>16}'.format('%8.2f' % getattr(self.dataOut, ds['variable'])[i][j])
876 line += '{:>16}'.format('%8.2f' % getattr(self.dataOut, ds['variable'])[i][j])
867 elif ds['nDim'] == 1:
877 elif ds['nDim'] == 1:
868 line += '{:>16}'.format('%8.2f' % getattr(self.dataOut, ds['variable'])[j])
878 line += '{:>16}'.format('%8.2f' % getattr(self.dataOut, ds['variable'])[j])
869 else:
879 else:
870 line += '{:>16}'.format('%8.2f' % getattr(self.dataOut, ds['variable']))
880 line += '{:>16}'.format('%8.2f' % getattr(self.dataOut, ds['variable']))
871
881
872 line += '\n'
882 line += '\n'
873 self.fp.write(line)
883 self.fp.write(line)
874
884
875 self.blockIndex += 1
885 self.blockIndex += 1
876 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
886 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
877
887
878 return
888 return
879
889
880 def closeFile(self):
890 def closeFile(self):
881
891
882 if self.fp:
892 if self.fp:
883 self.fp.close()
893 self.fp.close()
884
894
885 def close(self):
895 def close(self):
886
896
887 self.closeFile()
897 self.closeFile()
General Comments 0
You need to be logged in to leave comments. Login now