##// END OF EJS Templates
schain
RSS
tables was removed from imports
Miguel Valdez -
r639:87917d733682
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1005 +1,1004
1 1 import numpy
2 2 import time
3 3 import os
4 4 import h5py
5 5 import re
6 import tables
7 6
8 7 from schainpy.model.data.jrodata import *
9 8 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 9 from schainpy.model.io.jroIO_base import *
11 10
12 11
13 12 class HDF5Reader(ProcessingUnit):
14 13
15 14 ext = ".hdf5"
16 15
17 16 optchar = "D"
18 17
19 18 timezone = None
20 19
21 20 secStart = None
22 21
23 22 secEnd = None
24 23
25 24 fileIndex = None
26 25
27 26 blockIndex = None
28 27
29 28 blocksPerFile = None
30 29
31 30 path = None
32 31
33 32 #List of Files
34 33
35 34 filenameList = None
36 35
37 36 datetimeList = None
38 37
39 38 #Hdf5 File
40 39
41 40 fpMetadata = None
42 41
43 42 pathMeta = None
44 43
45 44 listMetaname = None
46 45
47 46 listMeta = None
48 47
49 48 listDataname = None
50 49
51 50 listData = None
52 51
53 52 listShapes = None
54 53
55 54 fp = None
56 55
57 56 #dataOut reconstruction
58 57
59 58 dataOut = None
60 59
61 60 nRecords = None
62 61
63 62
64 63 def __init__(self):
65 64 self.dataOut = self.__createObjByDefault()
66 65 return
67 66
68 67 def __createObjByDefault(self):
69 68
70 69 dataObj = Parameters()
71 70
72 71 return dataObj
73 72
74 73 def setup(self,path=None,
75 74 startDate=None,
76 75 endDate=None,
77 76 startTime=datetime.time(0,0,0),
78 77 endTime=datetime.time(23,59,59),
79 78 walk=True,
80 79 timezone='ut',
81 80 all=0,
82 81 online=False,
83 82 ext=None):
84 83
85 84 if ext==None:
86 85 ext = self.ext
87 86 self.timezone = timezone
88 87 # self.all = all
89 88 # self.online = online
90 89 self.path = path
91 90
92 91 startDateTime = datetime.datetime.combine(startDate,startTime)
93 92 endDateTime = datetime.datetime.combine(endDate,endTime)
94 93 secStart = (startDateTime-datetime.datetime(1970,1,1)).total_seconds()
95 94 secEnd = (endDateTime-datetime.datetime(1970,1,1)).total_seconds()
96 95
97 96 self.secStart = secStart
98 97 self.secEnd = secEnd
99 98
100 99 if not(online):
101 100 #Busqueda de archivos offline
102 101 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, secStart, secEnd, walk)
103 102 else:
104 103 self.__searchFilesOnline(path, walk)
105 104
106 105 if not(self.filenameList):
107 106 print "There is no files into the folder: %s"%(path)
108 107 sys.exit(-1)
109 108
110 109 # self.__getExpParameters()
111 110
112 111 self.fileIndex = -1
113 112
114 113 self.__setNextFileOffline()
115 114
116 115 self.__readMetadata()
117 116
118 117 self.blockIndex = 0
119 118
120 119 return
121 120
122 121 def __searchFilesOffline(self,
123 122 path,
124 123 startDate,
125 124 endDate,
126 125 ext,
127 126 startTime=datetime.time(0,0,0),
128 127 endTime=datetime.time(23,59,59),
129 128 secStart = 0,
130 129 secEnd = numpy.inf,
131 130 walk=True):
132 131
133 132 # self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
134 133 #
135 134 # self.__checkPath()
136 135 #
137 136 # self.__findDataForDates()
138 137 #
139 138 # self.__selectDataForTimes()
140 139 #
141 140 # for i in range(len(self.filenameList)):
142 141 # print "%s" %(self.filenameList[i])
143 142
144 143 pathList = []
145 144
146 145 if not walk:
147 146 #pathList.append(path)
148 147 multi_path = path.split(',')
149 148 for single_path in multi_path:
150 149 pathList.append(single_path)
151 150
152 151 else:
153 152 #dirList = []
154 153 multi_path = path.split(',')
155 154 for single_path in multi_path:
156 155 dirList = []
157 156 for thisPath in os.listdir(single_path):
158 157 if not os.path.isdir(os.path.join(single_path,thisPath)):
159 158 continue
160 159 if not isDoyFolder(thisPath):
161 160 continue
162 161
163 162 dirList.append(thisPath)
164 163
165 164 if not(dirList):
166 165 return None, None
167 166
168 167 thisDate = startDate
169 168
170 169 while(thisDate <= endDate):
171 170 year = thisDate.timetuple().tm_year
172 171 doy = thisDate.timetuple().tm_yday
173 172
174 173 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
175 174 if len(matchlist) == 0:
176 175 thisDate += datetime.timedelta(1)
177 176 continue
178 177 for match in matchlist:
179 178 pathList.append(os.path.join(single_path,match))
180 179
181 180 thisDate += datetime.timedelta(1)
182 181
183 182 if pathList == []:
184 183 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
185 184 return None, None
186 185
187 186 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
188 187
189 188 filenameList = []
190 189 datetimeList = []
191 190 pathDict = {}
192 191 filenameList_to_sort = []
193 192
194 193 for i in range(len(pathList)):
195 194
196 195 thisPath = pathList[i]
197 196
198 197 fileList = glob.glob1(thisPath, "*%s" %ext)
199 198 fileList.sort()
200 199 pathDict.setdefault(fileList[0])
201 200 pathDict[fileList[0]] = i
202 201 filenameList_to_sort.append(fileList[0])
203 202
204 203 filenameList_to_sort.sort()
205 204
206 205 for file in filenameList_to_sort:
207 206 thisPath = pathList[pathDict[file]]
208 207
209 208 fileList = glob.glob1(thisPath, "*%s" %ext)
210 209 fileList.sort()
211 210
212 211 for file in fileList:
213 212
214 213 filename = os.path.join(thisPath,file)
215 214 thisDatetime = self.__isFileinThisTime(filename, secStart, secEnd)
216 215
217 216 if not(thisDatetime):
218 217 continue
219 218
220 219 filenameList.append(filename)
221 220 datetimeList.append(thisDatetime)
222 221
223 222 if not(filenameList):
224 223 print "Any file was found for the time range %s - %s" %(startTime, endTime)
225 224 return None, None
226 225
227 226 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
228 227 print
229 228
230 229 for i in range(len(filenameList)):
231 230 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
232 231
233 232 self.filenameList = filenameList
234 233 self.datetimeList = datetimeList
235 234
236 235 return pathList, filenameList
237 236
238 237 def __isFileinThisTime(self, filename, startSeconds, endSeconds):
239 238 """
240 239 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
241 240
242 241 Inputs:
243 242 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
244 243
245 244 startTime : tiempo inicial del rango seleccionado en formato datetime.time
246 245
247 246 endTime : tiempo final del rango seleccionado en formato datetime.time
248 247
249 248 Return:
250 249 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
251 250 fecha especificado, de lo contrario retorna False.
252 251
253 252 Excepciones:
254 253 Si el archivo no existe o no puede ser abierto
255 254 Si la cabecera no puede ser leida.
256 255
257 256 """
258 257
259 258 try:
260 259 fp = fp = h5py.File(filename,'r')
261 260 except IOError:
262 261 traceback.print_exc()
263 262 raise IOError, "The file %s can't be opened" %(filename)
264 263
265 264 grp = fp['Data']
266 265 timeAux = grp['time']
267 266 time0 = timeAux[:][0].astype(numpy.float) #Time Vector
268 267
269 268 fp.close()
270 269
271 270 if self.timezone == 'lt':
272 271 time0 -= 5*3600
273 272
274 273 boolTimer = numpy.logical_and(time0 >= startSeconds,time0 < endSeconds)
275 274
276 275 if not (numpy.any(boolTimer)):
277 276 return None
278 277
279 278 thisDatetime = datetime.datetime.utcfromtimestamp(time0[0])
280 279 return thisDatetime
281 280
282 281 def __checkPath(self):
283 282 if os.path.exists(self.path):
284 283 self.status = 1
285 284 else:
286 285 self.status = 0
287 286 print 'Path:%s does not exists'%self.path
288 287
289 288 return
290 289
291 290 def __setNextFileOffline(self):
292 291 idFile = self.fileIndex
293 292 idFile += 1
294 293
295 294 if not(idFile < len(self.filenameList)):
296 295 print "No more Files"
297 296 return 0
298 297
299 298 filename = self.filenameList[idFile]
300 299
301 300 filePointer = h5py.File(filename,'r')
302 301
303 302 self.flagIsNewFile = 1
304 303 self.fileIndex = idFile
305 304 self.filename = filename
306 305
307 306 self.fp = filePointer
308 307
309 308 print "Setting the file: %s"%self.filename
310 309
311 310 self.__readMetadata()
312 311 self.__setBlockList()
313 312 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
314 313 self.nRecords = self.fp['Data'].attrs['nRecords']
315 314 self.blockIndex = 0
316 315 return 1
317 316
318 317 def __setBlockList(self):
319 318 '''
320 319 self.fp
321 320 self.startDateTime
322 321 self.endDateTime
323 322
324 323 self.blockList
325 324 self.blocksPerFile
326 325
327 326 '''
328 327 filePointer = self.fp
329 328 secStart = self.secStart
330 329 secEnd = self.secEnd
331 330
332 331 grp = filePointer['Data']
333 332 timeVector = grp['time'].value.astype(numpy.float)[0]
334 333
335 334 if self.timezone == 'lt':
336 335 timeVector -= 5*3600
337 336
338 337 ind = numpy.where(numpy.logical_and(timeVector >= secStart , timeVector < secEnd))[0]
339 338
340 339 self.blockList = ind
341 340 self.blocksPerFile = len(ind)
342 341
343 342 return
344 343
345 344 def __readMetadata(self):
346 345 '''
347 346 self.pathMeta
348 347
349 348 self.listShapes
350 349 self.listMetaname
351 350 self.listMeta
352 351
353 352 '''
354 353
355 354 grp = self.fp['Data']
356 355 pathMeta = os.path.join(self.path, grp.attrs['metadata'])
357 356
358 357 if pathMeta == self.pathMeta:
359 358 return
360 359 else:
361 360 self.pathMeta = pathMeta
362 361
363 362 filePointer = h5py.File(self.pathMeta,'r')
364 363 groupPointer = filePointer['Metadata']
365 364
366 365 listMetaname = []
367 366 listMetadata = []
368 367 for item in groupPointer.items():
369 368 name = item[0]
370 369
371 370 if name=='array dimensions':
372 371 table = groupPointer[name][:]
373 372 listShapes = {}
374 373 for shapes in table:
375 374 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4]])
376 375 else:
377 376 data = groupPointer[name].value
378 377 listMetaname.append(name)
379 378 listMetadata.append(data)
380 379
381 380 if name=='type':
382 381 self.__initDataOut(data)
383 382
384 383 filePointer.close()
385 384
386 385 self.listShapes = listShapes
387 386 self.listMetaname = listMetaname
388 387 self.listMeta = listMetadata
389 388
390 389 return
391 390
392 391 def __readData(self):
393 392 grp = self.fp['Data']
394 393 listdataname = []
395 394 listdata = []
396 395
397 396 for item in grp.items():
398 397 name = item[0]
399 398
400 399 if name == 'time':
401 400 listdataname.append('utctime')
402 401 timeAux = grp[name].value.astype(numpy.float)[0]
403 402 listdata.append(timeAux)
404 403 continue
405 404
406 405 listdataname.append(name)
407 406 array = self.__setDataArray(self.nRecords, grp[name],self.listShapes[name])
408 407 listdata.append(array)
409 408
410 409 self.listDataname = listdataname
411 410 self.listData = listdata
412 411 return
413 412
414 413 def __setDataArray(self, nRecords, dataset, shapes):
415 414
416 415 nChannels = shapes[0] #Dimension 0
417 416
418 417 nPoints = shapes[1] #Dimension 1, number of Points or Parameters
419 418
420 419 nSamples = shapes[2] #Dimension 2, number of samples or ranges
421 420
422 421 mode = shapes[3]
423 422
424 423 # if nPoints>1:
425 424 # arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
426 425 # else:
427 426 # arrayData = numpy.zeros((nRecords,nChannels,nSamples))
428 427 #
429 428 # chn = 'channel'
430 429 #
431 430 # for i in range(nChannels):
432 431 #
433 432 # data = dataset[chn + str(i)].value
434 433 #
435 434 # if nPoints>1:
436 435 # data = numpy.rollaxis(data,2)
437 436 #
438 437 # arrayData[:,i,:] = data
439 438
440 439 arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
441 440 doSqueeze = False
442 441 if mode == 0:
443 442 strds = 'channel'
444 443 nDatas = nChannels
445 444 newShapes = (nRecords,nPoints,nSamples)
446 445 if nPoints == 1:
447 446 doSqueeze = True
448 447 axisSqueeze = 2
449 448 else:
450 449 strds = 'param'
451 450 nDatas = nPoints
452 451 newShapes = (nRecords,nChannels,nSamples)
453 452 if nChannels == 1:
454 453 doSqueeze = True
455 454 axisSqueeze = 1
456 455
457 456 for i in range(nDatas):
458 457
459 458 data = dataset[strds + str(i)].value
460 459 data = data.reshape(newShapes)
461 460
462 461 if mode == 0:
463 462 arrayData[:,i,:,:] = data
464 463 else:
465 464 arrayData[:,:,i,:] = data
466 465
467 466 if doSqueeze:
468 467 arrayData = numpy.squeeze(arrayData, axis=axisSqueeze)
469 468
470 469 return arrayData
471 470
472 471 def __initDataOut(self, type):
473 472
474 473 # if type =='Parameters':
475 474 # self.dataOut = Parameters()
476 475 # elif type =='Spectra':
477 476 # self.dataOut = Spectra()
478 477 # elif type =='Voltage':
479 478 # self.dataOut = Voltage()
480 479 # elif type =='Correlation':
481 480 # self.dataOut = Correlation()
482 481
483 482 return
484 483
485 484 def __setDataOut(self):
486 485 listMeta = self.listMeta
487 486 listMetaname = self.listMetaname
488 487 listDataname = self.listDataname
489 488 listData = self.listData
490 489
491 490 blockIndex = self.blockIndex
492 491 blockList = self.blockList
493 492
494 493 for i in range(len(listMeta)):
495 494 setattr(self.dataOut,listMetaname[i],listMeta[i])
496 495
497 496 for j in range(len(listData)):
498 497 if listDataname[j]=='utctime':
499 498 # setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex]])
500 499 setattr(self.dataOut,'utctimeInit',listData[j][blockList[blockIndex]])
501 500 continue
502 501
503 502 setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex],:])
504 503
505 504 return self.dataOut.data_param
506 505
507 506 def getData(self):
508 507
509 508 # if self.flagNoMoreFiles:
510 509 # self.dataOut.flagNoData = True
511 510 # print 'Process finished'
512 511 # return 0
513 512 #
514 513 if self.blockIndex==self.blocksPerFile:
515 514 if not( self.__setNextFileOffline() ):
516 515 self.dataOut.flagNoData = True
517 516 return 0
518 517
519 518 #
520 519 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
521 520 # self.dataOut.flagNoData = True
522 521 # return 0
523 522
524 523 self.__readData()
525 524 self.__setDataOut()
526 525 self.dataOut.flagNoData = False
527 526
528 527 self.blockIndex += 1
529 528
530 529 return
531 530
532 531 def run(self, **kwargs):
533 532
534 533 if not(self.isConfig):
535 534 self.setup(**kwargs)
536 535 # self.setObjProperties()
537 536 self.isConfig = True
538 537
539 538 self.getData()
540 539
541 540 return
542 541
543 542 class HDF5Writer(Operation):
544 543
545 544 ext = ".hdf5"
546 545
547 546 optchar = "D"
548 547
549 548 metaoptchar = "M"
550 549
551 550 metaFile = None
552 551
553 552 filename = None
554 553
555 554 path = None
556 555
557 556 setFile = None
558 557
559 558 fp = None
560 559
561 560 grp = None
562 561
563 562 ds = None
564 563
565 564 firsttime = True
566 565
567 566 #Configurations
568 567
569 568 blocksPerFile = None
570 569
571 570 blockIndex = None
572 571
573 572 dataOut = None
574 573
575 574 #Data Arrays
576 575
577 576 dataList = None
578 577
579 578 metadataList = None
580 579
581 580 arrayDim = None
582 581
583 582 tableDim = None
584 583
585 584 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
586 585
587 586 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
588 587
589 588 mode = None
590 589
591 590 nDatas = None #Number of datasets to be stored per array
592 591
593 592 nDims = None #Number Dimensions in each dataset
594 593
595 594 nDimsForDs = None
596 595
597 596 def __init__(self):
598 597
599 598 Operation.__init__(self)
600 599 self.isConfig = False
601 600 return
602 601
603 602
604 603 def setup(self, dataOut, **kwargs):
605 604
606 605 self.path = kwargs['path']
607 606
608 607 if kwargs.has_key('ext'):
609 608 self.ext = kwargs['ext']
610 609
611 610 if kwargs.has_key('blocksPerFile'):
612 611 self.blocksPerFile = kwargs['blocksPerFile']
613 612 else:
614 613 self.blocksPerFile = 10
615 614
616 615 self.metadataList = kwargs['metadataList']
617 616
618 617 self.dataList = kwargs['dataList']
619 618
620 619 self.dataOut = dataOut
621 620
622 621 if kwargs.has_key('mode'):
623 622 mode = kwargs['mode']
624 623
625 624 if type(mode) == int:
626 625 mode = numpy.zeros(len(self.dataList)) + mode
627 626 else:
628 627 mode = numpy.zeros(len(self.dataList))
629 628
630 629 self.mode = mode
631 630
632 631 arrayDim = numpy.zeros((len(self.dataList),5))
633 632
634 633 #Table dimensions
635 634
636 635 dtype0 = self.dtype
637 636
638 637 tableList = []
639 638
640 639 for i in range(len(self.dataList)):
641 640
642 641 dataAux = getattr(self.dataOut, self.dataList[i])
643 642
644 643 if type(dataAux)==float or type(dataAux)==int:
645 644 arrayDim[i,0] = 1
646 645 else:
647 646 arrayDim0 = dataAux.shape
648 647 arrayDim[i,0] = len(arrayDim0)
649 648 arrayDim[i,4] = mode[i]
650 649
651 650 if len(arrayDim0) == 3:
652 651 arrayDim[i,1:-1] = numpy.array(arrayDim0)
653 652 elif len(arrayDim0) == 2:
654 653 arrayDim[i,2:-1] = numpy.array(arrayDim0) #nHeights
655 654 elif len(arrayDim0) == 1:
656 655 arrayDim[i,3] = arrayDim0
657 656 elif len(arrayDim0) == 0:
658 657 arrayDim[i,0] = 1
659 658 arrayDim[i,3] = 1
660 659
661 660 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
662 661 tableList.append(table)
663 662
664 663 self.arrayDim = arrayDim
665 664 self.tableDim = numpy.array(tableList, dtype = dtype0)
666 665 self.blockIndex = 0
667 666
668 667 return
669 668
670 669 def putMetadata(self):
671 670
672 671 fp = self.createMetadataFile()
673 672 self.writeMetadata(fp)
674 673 fp.close()
675 674 return
676 675
677 676 def createMetadataFile(self):
678 677 ext = self.ext
679 678 path = self.path
680 679 setFile = self.setFile
681 680
682 681 timeTuple = time.localtime(self.dataOut.utctime)
683 682
684 683 subfolder = ''
685 684 fullpath = os.path.join( path, subfolder )
686 685
687 686 if not( os.path.exists(fullpath) ):
688 687 os.mkdir(fullpath)
689 688 setFile = -1 #inicializo mi contador de seteo
690 689
691 690 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
692 691 fullpath = os.path.join( path, subfolder )
693 692
694 693 if not( os.path.exists(fullpath) ):
695 694 os.mkdir(fullpath)
696 695 setFile = -1 #inicializo mi contador de seteo
697 696
698 697 else:
699 698 filesList = os.listdir( fullpath )
700 699 filesList = sorted( filesList, key=str.lower )
701 700 if len( filesList ) > 0:
702 701 filesList = [k for k in filesList if 'M' in k]
703 702 filen = filesList[-1]
704 703 # el filename debera tener el siguiente formato
705 704 # 0 1234 567 89A BCDE (hex)
706 705 # x YYYY DDD SSS .ext
707 706 if isNumber( filen[8:11] ):
708 707 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
709 708 else:
710 709 setFile = -1
711 710 else:
712 711 setFile = -1 #inicializo mi contador de seteo
713 712
714 713 setFile += 1
715 714
716 715 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
717 716 timeTuple.tm_year,
718 717 timeTuple.tm_yday,
719 718 setFile,
720 719 ext )
721 720
722 721 filename = os.path.join( path, subfolder, file )
723 722 self.metaFile = file
724 723 #Setting HDF5 File
725 724 fp = h5py.File(filename,'w')
726 725
727 726 return fp
728 727
729 728 def writeMetadata(self, fp):
730 729
731 730 grp = fp.create_group("Metadata")
732 731 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
733 732
734 733 for i in range(len(self.metadataList)):
735 734 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
736 735 return
737 736
738 737 def setNextFile(self):
739 738
740 739 ext = self.ext
741 740 path = self.path
742 741 setFile = self.setFile
743 742 mode = self.mode
744 743
745 744 timeTuple = time.localtime(self.dataOut.utctime)
746 745 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
747 746
748 747 fullpath = os.path.join( path, subfolder )
749 748
750 749 if os.path.exists(fullpath):
751 750 filesList = os.listdir( fullpath )
752 751 filesList = [k for k in filesList if 'D' in k]
753 752 if len( filesList ) > 0:
754 753 filesList = sorted( filesList, key=str.lower )
755 754 filen = filesList[-1]
756 755 # el filename debera tener el siguiente formato
757 756 # 0 1234 567 89A BCDE (hex)
758 757 # x YYYY DDD SSS .ext
759 758 if isNumber( filen[8:11] ):
760 759 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
761 760 else:
762 761 setFile = -1
763 762 else:
764 763 setFile = -1 #inicializo mi contador de seteo
765 764
766 765 setFile += 1
767 766
768 767 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
769 768 timeTuple.tm_year,
770 769 timeTuple.tm_yday,
771 770 setFile,
772 771 ext )
773 772
774 773 filename = os.path.join( path, subfolder, file )
775 774
776 775 #Setting HDF5 File
777 776 fp = h5py.File(filename,'w')
778 777 grp = fp.create_group("Data")
779 778 grp.attrs['metadata'] = self.metaFile
780 779
781 780 # grp.attrs['blocksPerFile'] = 0
782 781
783 782 ds = []
784 783 data = []
785 784 nDimsForDs = []
786 785
787 786 nDatas = numpy.zeros(len(self.dataList))
788 787 nDims = self.arrayDim[:,0]
789 788
790 789 nDim1 = self.arrayDim[:,2]
791 790 nDim0 = self.arrayDim[:,3]
792 791
793 792 for i in range(len(self.dataList)):
794 793
795 794 if nDims[i]==1:
796 795 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
797 796 ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
798 797 ds.append(ds0)
799 798 data.append([])
800 799 nDimsForDs.append(nDims[i])
801 800 else:
802 801
803 802 if mode[i]==0:
804 803 strMode = "channel"
805 804 nDatas[i] = self.arrayDim[i,1]
806 805 else:
807 806 strMode = "param"
808 807 nDatas[i] = self.arrayDim[i,2]
809 808
810 809 if nDims[i]==2:
811 810 nDatas[i] = self.arrayDim[i,2]
812 811
813 812 grp0 = grp.create_group(self.dataList[i])
814 813
815 814 for j in range(int(nDatas[i])):
816 815 tableName = strMode + str(j)
817 816
818 817 if nDims[i] == 3:
819 818 ds0 = grp0.create_dataset(tableName, (nDim1[i],nDim0[i],1) , data = numpy.zeros((nDim1[i],nDim0[i],1)) ,maxshape=(None,nDim0[i],None), chunks=True)
820 819 else:
821 820 ds0 = grp0.create_dataset(tableName, (1,nDim0[i]), data = numpy.zeros((1,nDim0[i])) , maxshape=(None,nDim0[i]), chunks=True)
822 821
823 822 ds.append(ds0)
824 823 data.append([])
825 824 nDimsForDs.append(nDims[i])
826 825 self.nDatas = nDatas
827 826 self.nDims = nDims
828 827 self.nDimsForDs = nDimsForDs
829 828 #Saving variables
830 829 print 'Writing the file: %s'%filename
831 830 self.filename = filename
832 831 self.fp = fp
833 832 self.grp = grp
834 833 self.grp.attrs.modify('nRecords', 1)
835 834 self.ds = ds
836 835 self.data = data
837 836
838 837 self.setFile = setFile
839 838 self.firsttime = True
840 839 self.blockIndex = 0
841 840 return
842 841
843 842 def putData(self):
844 843
845 844 if not self.firsttime:
846 845 self.readBlock()
847 846
848 847 if self.blockIndex == self.blocksPerFile:
849 848
850 849 self.setNextFile()
851 850
852 851 self.setBlock()
853 852 self.writeBlock()
854 853
855 854 self.fp.flush()
856 855 self.fp.close()
857 856
858 857 return
859 858
860 859 def readBlock(self):
861 860
862 861 '''
863 862 data Array configured
864 863
865 864
866 865 self.data
867 866 '''
868 867 ds = self.ds
869 868 #Setting HDF5 File
870 869 fp = h5py.File(self.filename,'r+')
871 870 grp = fp["Data"]
872 871 ind = 0
873 872
874 873 # grp.attrs['blocksPerFile'] = 0
875 874 for i in range(len(self.dataList)):
876 875
877 876 if self.nDims[i]==1:
878 877 ds0 = grp[self.dataList[i]]
879 878 ds[ind] = ds0
880 879 ind += 1
881 880 else:
882 881 if self.mode[i]==0:
883 882 strMode = "channel"
884 883 else:
885 884 strMode = "param"
886 885
887 886 grp0 = grp[self.dataList[i]]
888 887
889 888 for j in range(int(self.nDatas[i])):
890 889 tableName = strMode + str(j)
891 890 ds0 = grp0[tableName]
892 891 ds[ind] = ds0
893 892 ind += 1
894 893
895 894
896 895 self.fp = fp
897 896 self.grp = grp
898 897 self.ds = ds
899 898
900 899 return
901 900
902 901
903 902 def setBlock(self):
904 903 '''
905 904 data Array configured
906 905
907 906
908 907 self.data
909 908 '''
910 909 #Creating Arrays
911 910 data = self.data
912 911 nDatas = self.nDatas
913 912 nDims = self.nDims
914 913 mode = self.mode
915 914 ind = 0
916 915
917 916 for i in range(len(self.dataList)):
918 917 dataAux = getattr(self.dataOut,self.dataList[i])
919 918
920 919 if nDims[i] == 1:
921 920 # data[ind] = numpy.array([str(dataAux)]).reshape((1,1))
922 921 data[ind] = dataAux
923 922 # if not self.firsttime:
924 923 # data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind]))
925 924 ind += 1
926 925 else:
927 926 for j in range(int(nDatas[i])):
928 927 if (mode[i] == 0) or (nDims[i] == 2): #In case division per channel or Dimensions is only 1
929 928 data[ind] = dataAux[j,:]
930 929 else:
931 930 data[ind] = dataAux[:,j,:]
932 931
933 932 # if nDims[i] == 3:
934 933 # data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1))
935 934
936 935 # if not self.firsttime:
937 936 # data[ind] = numpy.dstack((self.ds[ind][:], data[ind]))
938 937
939 938 # else:
940 939 # data[ind] = data[ind].reshape((1,data[ind].shape[0]))
941 940
942 941 # if not self.firsttime:
943 942 # data[ind] = numpy.vstack((self.ds[ind][:], data[ind]))
944 943 ind += 1
945 944
946 945 self.data = data
947 946 return
948 947
949 948 def writeBlock(self):
950 949 '''
951 950 Saves the block in the HDF5 file
952 951 '''
953 952 for i in range(len(self.ds)):
954 953 if self.firsttime:
955 954 # self.ds[i].resize(self.data[i].shape)
956 955 # self.ds[i][self.blockIndex,:] = self.data[i]
957 956 if type(self.data[i]) == numpy.ndarray:
958 957 nDims1 = len(self.ds[i].shape)
959 958
960 959 if nDims1 == 3:
961 960 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
962 961
963 962 self.ds[i].resize(self.data[i].shape)
964 963 self.ds[i][:] = self.data[i]
965 964 else:
966 965 if self.nDimsForDs[i] == 1:
967 966 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
968 967 self.ds[i][0,-1] = self.data[i]
969 968 elif self.nDimsForDs[i] == 2:
970 969 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
971 970 self.ds[i][self.blockIndex,:] = self.data[i]
972 971 elif self.nDimsForDs[i] == 3:
973 972
974 973 dataShape = self.data[i].shape
975 974 dsShape = self.ds[i].shape
976 975
977 976 if dataShape[0]==dsShape[0]:
978 977 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
979 978 self.ds[i][:,:,-1] = self.data[i]
980 979 else:
981 980 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1],self.ds[i].shape[2]))
982 981 self.ds[i][dsShape[0]:,:,0] = self.data[i]
983 982 # self.ds[i].append(self.data[i])
984 983 # self.fp.flush()
985 984 # if not self.firsttime:
986 985 # self.fp.root.Data._v_attrs.nRecords = self.blockIndex
987 986
988 987 # if self.firsttime:
989 988 # self.fp.close()
990 989 # self.readBlock2()
991 990
992 991 self.blockIndex += 1
993 992 self.firsttime = False
994 993 return
995 994
996 995 def run(self, dataOut, **kwargs):
997 996 if not(self.isConfig):
998 997 self.setup(dataOut, **kwargs)
999 998 self.isConfig = True
1000 999 self.putMetadata()
1001 1000 self.setNextFile()
1002 1001
1003 1002 self.putData()
1004 1003 return
1005 1004
General Comments 0
You need to be logged in to leave comments. Login now