##// END OF EJS Templates
schain
RSS
v3-devel similar to master with these changes.
imanay -
r1702:eaa588a03960
parent child
Show More
Side by Side Unified
Context file:
r1702:eaa588a03960 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,887 +1,897
1 1 import os
2 2 import time
3 3 import datetime
4 4
5 5 import numpy
6 6 import h5py
7 7
8 8 import schainpy.admin
9 9 from schainpy.model.data.jrodata import *
10 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 11 from schainpy.model.io.jroIO_base import *
12 12 from schainpy.utils import log
13 13
14 14
15 15 class HDFReader(Reader, ProcessingUnit):
16 16 """Processing unit to read HDF5 format files
17 17
18 18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 20 attributes.
21 21 It is possible to read any HDF5 file by given the structure in the `description`
22 22 parameter, also you can add extra values to metadata with the parameter `extras`.
23 23
24 24 Parameters:
25 25 -----------
26 26 path : str
27 27 Path where files are located.
28 28 startDate : date
29 29 Start date of the files
30 30 endDate : list
31 31 End date of the files
32 32 startTime : time
33 33 Start time of the files
34 34 endTime : time
35 35 End time of the files
36 36 description : dict, optional
37 37 Dictionary with the description of the HDF5 file
38 38 extras : dict, optional
39 39 Dictionary with extra metadata to be be added to `dataOut`
40 40
41 41 Examples
42 42 --------
43 43
44 44 desc = {
45 45 'Data': {
46 46 'data_output': ['u', 'v', 'w'],
47 47 'utctime': 'timestamps',
48 48 } ,
49 49 'Metadata': {
50 50 'heightList': 'heights'
51 51 }
52 52 }
53 53
54 54 desc = {
55 55 'Data': {
56 56 'data_output': 'winds',
57 57 'utctime': 'timestamps'
58 58 },
59 59 'Metadata': {
60 60 'heightList': 'heights'
61 61 }
62 62 }
63 63
64 64 extras = {
65 65 'timeZone': 300
66 66 }
67 67
68 68 reader = project.addReadUnit(
69 69 name='HDFReader',
70 70 path='/path/to/files',
71 71 startDate='2019/01/01',
72 72 endDate='2019/01/31',
73 73 startTime='00:00:00',
74 74 endTime='23:59:59',
75 75 # description=json.dumps(desc),
76 76 # extras=json.dumps(extras),
77 77 )
78 78
79 79 """
80 80
81 81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82 82
83 83 def __init__(self):
84 84 ProcessingUnit.__init__(self)
85 85 self.dataOut = Parameters()
86 86 self.ext = ".hdf5"
87 87 self.optchar = "D"
88 88 self.meta = {}
89 89 self.data = {}
90 90 self.open_file = h5py.File
91 91 self.open_mode = 'r'
92 92 self.description = {}
93 93 self.extras = {}
94 94 self.filefmt = "*%Y%j***"
95 95 self.folderfmt = "*%Y%j"
96 96 self.utcoffset = 0
97 97
98 98 def setup(self, **kwargs):
99 99
100 100 self.set_kwargs(**kwargs)
101 101 if not self.ext.startswith('.'):
102 102 self.ext = '.{}'.format(self.ext)
103 103
104 104 if self.online:
105 105 log.log("Searching files in online mode...", self.name)
106 106
107 107 for nTries in range(self.nTries):
108 108 fullpath = self.searchFilesOnLine(self.path, self.startDate,
109 109 self.endDate, self.expLabel, self.ext, self.walk,
110 110 self.filefmt, self.folderfmt)
111 111 try:
112 112 fullpath = next(fullpath)
113 113 except:
114 114 fullpath = None
115 115
116 116 if fullpath:
117 117 break
118 118
119 119 log.warning(
120 120 'Waiting {} sec for a valid file in {}: try {} ...'.format(
121 121 self.delay, self.path, nTries + 1),
122 122 self.name)
123 123 time.sleep(self.delay)
124 124
125 125 if not(fullpath):
126 126 raise schainpy.admin.SchainError(
127 127 'There isn\'t any valid file in {}'.format(self.path))
128 128
129 129 pathname, filename = os.path.split(fullpath)
130 130 self.year = int(filename[1:5])
131 131 self.doy = int(filename[5:8])
132 132 self.set = int(filename[8:11]) - 1
133 133 else:
134 134 log.log("Searching files in {}".format(self.path), self.name)
135 135 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
136 136 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
137 137
138 138 self.setNextFile()
139 139
140 140 return
141 141
142 142 def readFirstHeader(self):
143 143 '''Read metadata and data'''
144 144
145 145 self.__readMetadata()
146 146 self.__readData()
147 self.__setBlockList()
148
147 self.__setBlockList()
148 # similar to master
149 if 'type' in self.meta:
150 self.dataOut = eval(self.meta['type'])()
151 # similar to master
149 152 for attr in self.meta:
150 153 setattr(self.dataOut, attr, self.meta[attr])
151 154
152 155 self.blockIndex = 0
153 156
154 157 return
155 158
156 159 def __setBlockList(self):
157 160 '''
158 161 Selects the data within the times defined
159 162
160 163 self.fp
161 164 self.startTime
162 165 self.endTime
163 166 self.blockList
164 167 self.blocksPerFile
165 168
166 169 '''
167 170
168 171 startTime = self.startTime
169 172 endTime = self.endTime
170 173 thisUtcTime = self.data['utctime'] + self.utcoffset
171 174 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
172 175 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
173 176
174 177 thisDate = thisDatetime.date()
175 178 thisTime = thisDatetime.time()
176 179
177 180 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
178 181 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
179 182
180 183 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
181 184
182 185 self.blockList = ind
183 186 self.blocksPerFile = len(ind)
187 # similar to master
188 if len(ind)==0:
189 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.blockIndex,
190 self.blocksPerFile,
191 thisDatetime))
192 self.setNextFile()
193 # similar to master
184 194 return
185 195
186 196 def __readMetadata(self):
187 197 '''
188 198 Reads Metadata
189 199 '''
190 200
191 201 meta = {}
192 202
193 203 if self.description:
194 204 for key, value in self.description['Metadata'].items():
195 205 meta[key] = self.fp[value][()]
196 206 else:
197 207 grp = self.fp['Metadata']
198 208 for name in grp:
199 209 meta[name] = grp[name][()]
200 210
201 211 if self.extras:
202 212 for key, value in self.extras.items():
203 213 meta[key] = value
204 214 self.meta = meta
205 215
206 216 return
207 217
208 218 def __readData(self):
209 219
210 220 data = {}
211 221
212 222 if self.description:
213 223 for key, value in self.description['Data'].items():
214 224 if isinstance(value, str):
215 225 if isinstance(self.fp[value], h5py.Dataset):
216 226 data[key] = self.fp[value][()]
217 227 elif isinstance(self.fp[value], h5py.Group):
218 228 array = []
219 229 for ch in self.fp[value]:
220 230 array.append(self.fp[value][ch][()])
221 231 data[key] = numpy.array(array)
222 232 elif isinstance(value, list):
223 233 array = []
224 234 for ch in value:
225 235 array.append(self.fp[ch][()])
226 236 data[key] = numpy.array(array)
227 237 else:
228 238 grp = self.fp['Data']
229 239 for name in grp:
230 240 if isinstance(grp[name], h5py.Dataset):
231 241 array = grp[name][()]
232 242 elif isinstance(grp[name], h5py.Group):
233 243 array = []
234 244 for ch in grp[name]:
235 245 array.append(grp[name][ch][()])
236 246 array = numpy.array(array)
237 247 else:
238 248 log.warning('Unknown type: {}'.format(name))
239 249
240 250 if name in self.description:
241 251 key = self.description[name]
242 252 else:
243 253 key = name
244 254 data[key] = array
245 255
246 256 self.data = data
247 257 return
248 258
249 259 def getData(self):
250 260
251 261 for attr in self.data:
252 262 if self.data[attr].ndim == 1:
253 263 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
254 264 else:
255 265 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
256 266
257 267 self.dataOut.flagNoData = False
258 268 self.blockIndex += 1
259 269
260 270 log.log("Block No. {}/{} -> {}".format(
261 271 self.blockIndex,
262 272 self.blocksPerFile,
263 273 self.dataOut.datatime.ctime()), self.name)
264 274
265 275 return
266 276
267 277 def run(self, **kwargs):
268 278
269 279 if not(self.isConfig):
270 280 self.setup(**kwargs)
271 281 self.isConfig = True
272 282
273 283 if self.blockIndex == self.blocksPerFile:
274 284 self.setNextFile()
275 285
276 286 self.getData()
277
287 ''' # this block is missing in master.
278 288 if 'type' in self.meta:
279 289 self.dataOut.type = self.meta['type'].decode('utf-8')
280
290 '''
281 291 return
282 292
283 293 @MPDecorator
284 294 class HDFWriter(Operation):
285 295 """Operation to write HDF5 files.
286 296
287 297 The HDF5 file contains by default two groups Data and Metadata where
288 298 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
289 299 parameters, data attributes are normaly time dependent where the metadata
290 300 are not.
291 301 It is possible to customize the structure of the HDF5 file with the
292 302 optional description parameter see the examples.
293 303
294 304 Parameters:
295 305 -----------
296 306 path : str
297 307 Path where files will be saved.
298 308 blocksPerFile : int
299 309 Number of blocks per file
300 310 metadataList : list
301 311 List of the dataOut attributes that will be saved as metadata
302 312 dataList : int
303 313 List of the dataOut attributes that will be saved as data
304 314 setType : bool
305 315 If True the name of the files corresponds to the timestamp of the data
306 316 description : dict, optional
307 317 Dictionary with the desired description of the HDF5 file
308 318
309 319 Examples
310 320 --------
311 321
312 322 desc = {
313 323 'data_output': {'winds': ['z', 'w', 'v']},
314 324 'utctime': 'timestamps',
315 325 'heightList': 'heights'
316 326 }
317 327 desc = {
318 328 'data_output': ['z', 'w', 'v'],
319 329 'utctime': 'timestamps',
320 330 'heightList': 'heights'
321 331 }
322 332 desc = {
323 333 'Data': {
324 334 'data_output': 'winds',
325 335 'utctime': 'timestamps'
326 336 },
327 337 'Metadata': {
328 338 'heightList': 'heights'
329 339 }
330 340 }
331 341
332 342 writer = proc_unit.addOperation(name='HDFWriter')
333 343 writer.addParameter(name='path', value='/path/to/file')
334 344 writer.addParameter(name='blocksPerFile', value='32')
335 345 writer.addParameter(name='metadataList', value='heightList,timeZone')
336 346 writer.addParameter(name='dataList',value='data_output,utctime')
337 347 # writer.addParameter(name='description',value=json.dumps(desc))
338 348
339 349 """
340 350
341 351 ext = ".hdf5"
342 352 optchar = "D"
343 353 filename = None
344 354 path = None
345 355 setFile = None
346 356 fp = None
347 357 firsttime = True
348 358 # Configurations
349 359 blocksPerFile = None
350 360 blockIndex = None
351 361 dataOut = None
352 362 # Data Arrays
353 363 dataList = None
354 364 metadataList = None
355 365 currentDay = None
356 366 lastTime = None
357 367
358 368 def __init__(self):
359 369
360 370 Operation.__init__(self)
361 371 return
362 372
363 373 def set_kwargs(self, **kwargs):
364 374
365 375 for key, value in kwargs.items():
366 376 setattr(self, key, value)
367 377
368 378 def set_kwargs_obj(self, obj, **kwargs):
369 379
370 380 for key, value in kwargs.items():
371 381 setattr(obj, key, value)
372 382
373 383 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None, **kwargs):
374 384 self.path = path
375 385 self.blocksPerFile = blocksPerFile
376 386 self.metadataList = metadataList
377 387 self.dataList = [s.strip() for s in dataList]
378 388 self.setType = setType
379 389 self.description = description
380 390 self.set_kwargs(**kwargs)
381 391
382 392 if self.metadataList is None:
383 393 self.metadataList = self.dataOut.metadata_list
384 394
385 395 tableList = []
386 396 dsList = []
387 397
388 398 for i in range(len(self.dataList)):
389 399 dsDict = {}
390 400 if hasattr(self.dataOut, self.dataList[i]):
391 401 dataAux = getattr(self.dataOut, self.dataList[i])
392 402 dsDict['variable'] = self.dataList[i]
393 403 else:
394 404 log.warning('Attribute {} not found in dataOut', self.name)
395 405 continue
396 406
397 407 if dataAux is None:
398 408 continue
399 409 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float32)):
400 410 dsDict['nDim'] = 0
401 411 else:
402 412 dsDict['nDim'] = len(dataAux.shape)
403 413 dsDict['shape'] = dataAux.shape
404 414 dsDict['dsNumber'] = dataAux.shape[0]
405 415 dsDict['dtype'] = dataAux.dtype
406 416
407 417 dsList.append(dsDict)
408 418
409 419 self.dsList = dsList
410 420 self.currentDay = self.dataOut.datatime.date()
411 421
412 422 def timeFlag(self):
413 423 currentTime = self.dataOut.utctime
414 424 timeTuple = time.localtime(currentTime)
415 425 dataDay = timeTuple.tm_yday
416 426
417 427 if self.lastTime is None:
418 428 self.lastTime = currentTime
419 429 self.currentDay = dataDay
420 430 return False
421 431
422 432 timeDiff = currentTime - self.lastTime
423 433
424 434 # Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
425 435 if dataDay != self.currentDay:
426 436 self.currentDay = dataDay
427 437 return True
428 438 elif timeDiff > 3 * 60 * 60:
429 439 self.lastTime = currentTime
430 440 return True
431 441 else:
432 442 self.lastTime = currentTime
433 443 return False
434 444
435 445 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
436 446 dataList=[], setType=None, description={}, **kwargs):
437 447
438 448 self.dataOut = dataOut
439 449 self.set_kwargs_obj(self.dataOut, **kwargs)
440 450 if not(self.isConfig):
441 451 self.setup(path=path, blocksPerFile=blocksPerFile,
442 452 metadataList=metadataList, dataList=dataList,
443 453 setType=setType, description=description, **kwargs)
444 454
445 455 self.isConfig = True
446 456 self.setNextFile()
447 457
448 458 self.putData()
449 459 return
450 460
451 461 def setNextFile(self):
452 462
453 463 ext = self.ext
454 464 path = self.path
455 465 setFile = self.setFile
456 466
457 467 timeTuple = time.localtime(self.dataOut.utctime)
458 468 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
459 469 fullpath = os.path.join(path, subfolder)
460 470
461 471 if os.path.exists(fullpath):
462 472 filesList = os.listdir(fullpath)
463 473 filesList = [k for k in filesList if k.startswith(self.optchar)]
464 474 if len(filesList) > 0:
465 475 filesList = sorted(filesList, key=str.lower)
466 476 filen = filesList[-1]
467 477 # el filename debera tener el siguiente formato
468 478 # 0 1234 567 89A BCDE (hex)
469 479 # x YYYY DDD SSS .ext
470 480 if isNumber(filen[8:11]):
471 481 setFile = int(filen[8:11]) # inicializo mi contador de seteo al seteo del ultimo file
472 482 else:
473 483 setFile = -1
474 484 else:
475 485 setFile = -1 # inicializo mi contador de seteo
476 486 else:
477 487 os.makedirs(fullpath)
478 488 setFile = -1 # inicializo mi contador de seteo
479 489
480 490 if self.setType is None:
481 491 setFile += 1
482 492 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
483 493 timeTuple.tm_year,
484 494 timeTuple.tm_yday,
485 495 setFile,
486 496 ext)
487 497 else:
488 498 setFile = timeTuple.tm_hour * 60 + timeTuple.tm_min
489 499 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
490 500 timeTuple.tm_year,
491 501 timeTuple.tm_yday,
492 502 setFile,
493 503 ext)
494 504
495 505 self.filename = os.path.join(path, subfolder, file)
496 506
497 507 # Setting HDF5 File
498 508 self.fp = h5py.File(self.filename, 'w')
499 509 # write metadata
500 510 self.writeMetadata(self.fp)
501 511 # Write data
502 512 self.writeData(self.fp)
503 513
504 514 def getLabel(self, name, x=None):
505 515
506 516 if x is None:
507 517 if 'Data' in self.description:
508 518 data = self.description['Data']
509 519 if 'Metadata' in self.description:
510 520 data.update(self.description['Metadata'])
511 521 else:
512 522 data = self.description
513 523 if name in data:
514 524 if isinstance(data[name], str):
515 525 return data[name]
516 526 elif isinstance(data[name], list):
517 527 return None
518 528 elif isinstance(data[name], dict):
519 529 for key, value in data[name].items():
520 530 return key
521 531 return name
522 532 else:
523 533 if 'Data' in self.description:
524 534 data = self.description['Data']
525 535 if 'Metadata' in self.description:
526 536 data.update(self.description['Metadata'])
527 537 else:
528 538 data = self.description
529 539 if name in data:
530 540 if isinstance(data[name], list):
531 541 return data[name][x]
532 542 elif isinstance(data[name], dict):
533 543 for key, value in data[name].items():
534 544 return value[x]
535 545 if 'cspc' in name:
536 546 return 'pair{:02d}'.format(x)
537 547 else:
538 548 return 'channel{:02d}'.format(x)
539 549
540 550 def writeMetadata(self, fp):
541 551
542 552 if self.description:
543 553 if 'Metadata' in self.description:
544 554 grp = fp.create_group('Metadata')
545 555 else:
546 556 grp = fp
547 557 else:
548 558 grp = fp.create_group('Metadata')
549 559
550 560 for i in range(len(self.metadataList)):
551 561 if not hasattr(self.dataOut, self.metadataList[i]):
552 562 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
553 563 continue
554 564 value = getattr(self.dataOut, self.metadataList[i])
555 565 if isinstance(value, bool):
556 566 if value is True:
557 567 value = 1
558 568 else:
559 569 value = 0
560 570 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
561 571 return
562 572
563 573 def writeData(self, fp):
564 574
565 575 if self.description:
566 576 if 'Data' in self.description:
567 577 grp = fp.create_group('Data')
568 578 else:
569 579 grp = fp
570 580 else:
571 581 grp = fp.create_group('Data')
572 582
573 583 dtsets = []
574 584 data = []
575 585
576 586 for dsInfo in self.dsList:
577 587 if dsInfo['nDim'] == 0:
578 588 ds = grp.create_dataset(
579 589 self.getLabel(dsInfo['variable']),
580 590 (self.blocksPerFile,),
581 591 chunks=True,
582 592 dtype=numpy.float64)
583 593 dtsets.append(ds)
584 594 data.append((dsInfo['variable'], -1))
585 595 else:
586 596 label = self.getLabel(dsInfo['variable'])
587 597 if label is not None:
588 598 sgrp = grp.create_group(label)
589 599 else:
590 600 sgrp = grp
591 601 for i in range(dsInfo['dsNumber']):
592 602 ds = sgrp.create_dataset(
593 603 self.getLabel(dsInfo['variable'], i),
594 604 (self.blocksPerFile,) + dsInfo['shape'][1:],
595 605 chunks=True,
596 606 dtype=dsInfo['dtype'])
597 607 dtsets.append(ds)
598 608 data.append((dsInfo['variable'], i))
599 609 fp.flush()
600 610
601 611 log.log('Creating file: {}'.format(fp.filename), self.name)
602 612
603 613 self.ds = dtsets
604 614 self.data = data
605 615 self.firsttime = True
606 616 self.blockIndex = 0
607 617 return
608 618
609 619 def putData(self):
610 620
611 621 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
612 622 self.closeFile()
613 623 self.setNextFile()
614 624
615 625 for i, ds in enumerate(self.ds):
616 626 attr, ch = self.data[i]
617 627 if ch == -1:
618 628 ds[self.blockIndex] = getattr(self.dataOut, attr)
619 629 else:
620 630 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
621 631
622 632 self.fp.flush()
623 633 self.blockIndex += 1
624 634 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
625 635
626 636 return
627 637
628 638 def closeFile(self):
629 639
630 640 if self.blockIndex != self.blocksPerFile:
631 641 for ds in self.ds:
632 642 ds.resize(self.blockIndex, axis=0)
633 643
634 644 if self.fp:
635 645 self.fp.flush()
636 646 self.fp.close()
637 647
638 648 def close(self):
639 649
640 650 self.closeFile()
641 651
642 652
643 653 @MPDecorator
644 654 class ASCIIWriter(Operation):
645 655 """Operation to write data in ascii files.
646 656
647 657 Parameters:
648 658 -----------
649 659 path : str
650 660 Path where files will be saved.
651 661 blocksPerFile : int
652 662 Number of blocks per file
653 663 metadataList : list
654 664 List of the dataOut attributes that will be saved as metadata
655 665 dataDict : dict
656 666 Dictionary with the varaibles to be saved
657 667 setType : bool
658 668 If True the name of the files corresponds to the timestamp of the data
659 669
660 670 Examples
661 671 --------
662 672
663 673 data = {
664 674 'data_output': ['z', 'w', 'v'],
665 675 'utctime': 'time',
666 676 'heightList': 'height'
667 677 }
668 678
669 679 writer = proc_unit.addOperation(name='ASCIIWriter')
670 680 writer.addParameter(name='path', value='/path/to/file')
671 681 writer.addParameter(name='blocksPerFile', value='32')
672 682 writer.addParameter(name='dataDict',value=json.dumps(data))
673 683
674 684 """
675 685
676 686 ext = ".txt"
677 687 optchar = "D"
678 688 filename = None
679 689 path = None
680 690 setFile = None
681 691 fp = None
682 692 firsttime = True
683 693 # Configurations
684 694 blocksPerFile = None
685 695 blockIndex = None
686 696 dataOut = None
687 697 # Data Arrays
688 698 dataDict = None
689 699 metadataList = None
690 700 currentDay = None
691 701 lastTime = None
692 702 localtime = True
693 703
694 704 def __init__(self):
695 705
696 706 Operation.__init__(self)
697 707 return
698 708
699 709 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataDict=None, setType=None, localtime=True):
700 710 self.path = path
701 711 self.blocksPerFile = blocksPerFile
702 712 self.metadataList = metadataList
703 713 self.dataDict = dataDict
704 714 self.setType = setType
705 715 self.localtime = localtime
706 716
707 717 if self.metadataList is None:
708 718 self.metadataList = self.dataOut.metadata_list
709 719
710 720 dsList = []
711 721
712 722 for key, value in self.dataDict.items():
713 723 dsDict = {}
714 724 if hasattr(self.dataOut, key):
715 725 dataAux = getattr(self.dataOut, key)
716 726 dsDict['variable'] = key
717 727 else:
718 728 log.warning('Attribute {} not found in dataOut', self.name)
719 729 continue
720 730
721 731 if dataAux is None:
722 732 continue
723 733 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float32)):
724 734 dsDict['nDim'] = 0
725 735 else:
726 736 dsDict['nDim'] = len(dataAux.shape)
727 737 dsDict['shape'] = dataAux.shape
728 738 dsDict['dsNumber'] = dataAux.shape[0]
729 739 dsDict['dtype'] = dataAux.dtype
730 740
731 741 dsList.append(dsDict)
732 742 self.dsList = dsList
733 743 self.currentDay = self.dataOut.datatime.date()
734 744
735 745 def timeFlag(self):
736 746 currentTime = self.dataOut.utctime
737 747 if self.localtime:
738 748 timeTuple = time.localtime(currentTime)
739 749 else:
740 750 timeTuple = time.gmtime(currentTime)
741 751
742 752 dataDay = timeTuple.tm_yday
743 753
744 754 if self.lastTime is None:
745 755 self.lastTime = currentTime
746 756 self.currentDay = dataDay
747 757 return False
748 758
749 759 timeDiff = currentTime - self.lastTime
750 760
751 761 # Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
752 762 if dataDay != self.currentDay:
753 763 self.currentDay = dataDay
754 764 return True
755 765 elif timeDiff > 3 * 60 * 60:
756 766 self.lastTime = currentTime
757 767 return True
758 768 else:
759 769 self.lastTime = currentTime
760 770 return False
761 771
762 772 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
763 773 dataDict={}, setType=None, localtime=True):
764 774
765 775 self.dataOut = dataOut
766 776 if not(self.isConfig):
767 777 self.setup(path=path, blocksPerFile=blocksPerFile,
768 778 metadataList=metadataList, dataDict=dataDict,
769 779 setType=setType, localtime=localtime)
770 780
771 781 self.isConfig = True
772 782 self.setNextFile()
773 783
774 784 self.putData()
775 785 return
776 786
777 787 def setNextFile(self):
778 788
779 789 ext = self.ext
780 790 path = self.path
781 791 setFile = self.setFile
782 792 if self.localtime:
783 793 timeTuple = time.localtime(self.dataOut.utctime)
784 794 else:
785 795 timeTuple = time.gmtime(self.dataOut.utctime)
786 796 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
787 797 fullpath = os.path.join(path, subfolder)
788 798
789 799 if os.path.exists(fullpath):
790 800 filesList = os.listdir(fullpath)
791 801 filesList = [k for k in filesList if k.startswith(self.optchar)]
792 802 if len(filesList) > 0:
793 803 filesList = sorted(filesList, key=str.lower)
794 804 filen = filesList[-1]
795 805 # el filename debera tener el siguiente formato
796 806 # 0 1234 567 89A BCDE (hex)
797 807 # x YYYY DDD SSS .ext
798 808 if isNumber(filen[8:11]):
799 809 setFile = int(filen[8:11]) # inicializo mi contador de seteo al seteo del ultimo file
800 810 else:
801 811 setFile = -1
802 812 else:
803 813 setFile = -1 # inicializo mi contador de seteo
804 814 else:
805 815 os.makedirs(fullpath)
806 816 setFile = -1 # inicializo mi contador de seteo
807 817
808 818 if self.setType is None:
809 819 setFile += 1
810 820 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
811 821 timeTuple.tm_year,
812 822 timeTuple.tm_yday,
813 823 setFile,
814 824 ext)
815 825 else:
816 826 setFile = timeTuple.tm_hour * 60 + timeTuple.tm_min
817 827 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
818 828 timeTuple.tm_year,
819 829 timeTuple.tm_yday,
820 830 setFile,
821 831 ext)
822 832
823 833 self.filename = os.path.join(path, subfolder, file)
824 834
825 835 # Setting HDF5 File
826 836 self.fp = open(self.filename, 'w')
827 837 # write metadata
828 838 self.writeMetadata(self.fp)
829 839 # Write data
830 840 self.writeData(self.fp)
831 841
832 842 def writeMetadata(self, fp):
833 843
834 844 line = ''
835 845 for d in self.dsList:
836 846 par = self.dataDict[d['variable']]
837 847 if isinstance(par, (list,tuple)):
838 848 for p in par:
839 849 line += '{:>16}'.format(p)
840 850 else:
841 851 line += '{:>16}'.format(par)
842 852
843 853 line += '\n'
844 854 fp.write(line)
845 855
846 856 def writeData(self, fp):
847 857
848 858 log.log('Creating file: {}'.format(self.filename), self.name)
849 859
850 860 self.firsttime = True
851 861 self.blockIndex = 0
852 862 return
853 863
854 864 def putData(self):
855 865
856 866 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
857 867 self.closeFile()
858 868 self.setNextFile()
859 869
860 870 line = ''
861 871 for j in range(len(self.dataOut.heightList)):
862 872 for ds in self.dsList:
863 873 par = self.dataDict[ds['variable']]
864 874 if ds['nDim'] == 2:
865 875 for i in range(len(par)):
866 876 line += '{:>16}'.format('%8.2f' % getattr(self.dataOut, ds['variable'])[i][j])
867 877 elif ds['nDim'] == 1:
868 878 line += '{:>16}'.format('%8.2f' % getattr(self.dataOut, ds['variable'])[j])
869 879 else:
870 880 line += '{:>16}'.format('%8.2f' % getattr(self.dataOut, ds['variable']))
871 881
872 882 line += '\n'
873 883 self.fp.write(line)
874 884
875 885 self.blockIndex += 1
876 886 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
877 887
878 888 return
879 889
880 890 def closeFile(self):
881 891
882 892 if self.fp:
883 893 self.fp.close()
884 894
885 895 def close(self):
886 896
887 897 self.closeFile()
General Comments 0
You need to be logged in to leave comments. Login now