##// END OF EJS Templates
schain
RSS
-Bug fixed in HDF5 writing module...
Julio Valdez -
r724:fb11662812fb
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1004 +1,1026
1 1 import numpy
2 2 import time
3 3 import os
4 4 import h5py
5 5 import re
6 6
7 7 from schainpy.model.data.jrodata import *
8 8 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 9 from schainpy.model.io.jroIO_base import *
10 10
11 11
12 12 class HDF5Reader(ProcessingUnit):
13 13
14 14 ext = ".hdf5"
15 15
16 16 optchar = "D"
17 17
18 18 timezone = None
19 19
20 20 secStart = None
21 21
22 22 secEnd = None
23 23
24 24 fileIndex = None
25 25
26 26 blockIndex = None
27 27
28 28 blocksPerFile = None
29 29
30 30 path = None
31 31
32 32 #List of Files
33 33
34 34 filenameList = None
35 35
36 36 datetimeList = None
37 37
38 38 #Hdf5 File
39 39
40 40 fpMetadata = None
41 41
42 42 pathMeta = None
43 43
44 44 listMetaname = None
45 45
46 46 listMeta = None
47 47
48 48 listDataname = None
49 49
50 50 listData = None
51 51
52 52 listShapes = None
53 53
54 54 fp = None
55 55
56 56 #dataOut reconstruction
57 57
58 58 dataOut = None
59 59
60 60 nRecords = None
61 61
62 62
63 63 def __init__(self):
64 64 self.dataOut = self.__createObjByDefault()
65 65 return
66 66
67 67 def __createObjByDefault(self):
68 68
69 69 dataObj = Parameters()
70 70
71 71 return dataObj
72 72
73 73 def setup(self,path=None,
74 74 startDate=None,
75 75 endDate=None,
76 76 startTime=datetime.time(0,0,0),
77 77 endTime=datetime.time(23,59,59),
78 78 walk=True,
79 79 timezone='ut',
80 80 all=0,
81 81 online=False,
82 82 ext=None):
83 83
84 84 if ext==None:
85 85 ext = self.ext
86 86 self.timezone = timezone
87 87 # self.all = all
88 88 # self.online = online
89 89 self.path = path
90 90
91 91 startDateTime = datetime.datetime.combine(startDate,startTime)
92 92 endDateTime = datetime.datetime.combine(endDate,endTime)
93 93 secStart = (startDateTime-datetime.datetime(1970,1,1)).total_seconds()
94 94 secEnd = (endDateTime-datetime.datetime(1970,1,1)).total_seconds()
95 95
96 96 self.secStart = secStart
97 97 self.secEnd = secEnd
98 98
99 99 if not(online):
100 100 #Busqueda de archivos offline
101 101 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, secStart, secEnd, walk)
102 102 else:
103 103 self.__searchFilesOnline(path, walk)
104 104
105 105 if not(self.filenameList):
106 106 print "There is no files into the folder: %s"%(path)
107 107 sys.exit(-1)
108 108
109 109 # self.__getExpParameters()
110 110
111 111 self.fileIndex = -1
112 112
113 113 self.__setNextFileOffline()
114 114
115 115 self.__readMetadata()
116 116
117 117 self.blockIndex = 0
118 118
119 119 return
120 120
121 121 def __searchFilesOffline(self,
122 122 path,
123 123 startDate,
124 124 endDate,
125 125 ext,
126 126 startTime=datetime.time(0,0,0),
127 127 endTime=datetime.time(23,59,59),
128 128 secStart = 0,
129 129 secEnd = numpy.inf,
130 130 walk=True):
131 131
132 132 # self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
133 133 #
134 134 # self.__checkPath()
135 135 #
136 136 # self.__findDataForDates()
137 137 #
138 138 # self.__selectDataForTimes()
139 139 #
140 140 # for i in range(len(self.filenameList)):
141 141 # print "%s" %(self.filenameList[i])
142 142
143 143 pathList = []
144 144
145 145 if not walk:
146 146 #pathList.append(path)
147 147 multi_path = path.split(',')
148 148 for single_path in multi_path:
149 149 pathList.append(single_path)
150 150
151 151 else:
152 152 #dirList = []
153 153 multi_path = path.split(',')
154 154 for single_path in multi_path:
155 155 dirList = []
156 156 for thisPath in os.listdir(single_path):
157 157 if not os.path.isdir(os.path.join(single_path,thisPath)):
158 158 continue
159 159 if not isDoyFolder(thisPath):
160 160 continue
161 161
162 162 dirList.append(thisPath)
163 163
164 164 if not(dirList):
165 165 return None, None
166 166
167 167 thisDate = startDate
168 168
169 169 while(thisDate <= endDate):
170 170 year = thisDate.timetuple().tm_year
171 171 doy = thisDate.timetuple().tm_yday
172 172
173 173 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
174 174 if len(matchlist) == 0:
175 175 thisDate += datetime.timedelta(1)
176 176 continue
177 177 for match in matchlist:
178 178 pathList.append(os.path.join(single_path,match))
179 179
180 180 thisDate += datetime.timedelta(1)
181 181
182 182 if pathList == []:
183 183 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
184 184 return None, None
185 185
186 186 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
187 187
188 188 filenameList = []
189 189 datetimeList = []
190 190 pathDict = {}
191 191 filenameList_to_sort = []
192 192
193 193 for i in range(len(pathList)):
194 194
195 195 thisPath = pathList[i]
196 196
197 197 fileList = glob.glob1(thisPath, "*%s" %ext)
198 198 fileList.sort()
199 199 pathDict.setdefault(fileList[0])
200 200 pathDict[fileList[0]] = i
201 201 filenameList_to_sort.append(fileList[0])
202 202
203 203 filenameList_to_sort.sort()
204 204
205 205 for file in filenameList_to_sort:
206 206 thisPath = pathList[pathDict[file]]
207 207
208 208 fileList = glob.glob1(thisPath, "*%s" %ext)
209 209 fileList.sort()
210 210
211 211 for file in fileList:
212 212
213 213 filename = os.path.join(thisPath,file)
214 214 thisDatetime = self.__isFileinThisTime(filename, secStart, secEnd)
215 215
216 216 if not(thisDatetime):
217 217 continue
218 218
219 219 filenameList.append(filename)
220 220 datetimeList.append(thisDatetime)
221 221
222 222 if not(filenameList):
223 223 print "Any file was found for the time range %s - %s" %(startTime, endTime)
224 224 return None, None
225 225
226 226 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
227 227 print
228 228
229 229 for i in range(len(filenameList)):
230 230 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
231 231
232 232 self.filenameList = filenameList
233 233 self.datetimeList = datetimeList
234 234
235 235 return pathList, filenameList
236 236
237 237 def __isFileinThisTime(self, filename, startSeconds, endSeconds):
238 238 """
239 239 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
240 240
241 241 Inputs:
242 242 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
243 243
244 244 startTime : tiempo inicial del rango seleccionado en formato datetime.time
245 245
246 246 endTime : tiempo final del rango seleccionado en formato datetime.time
247 247
248 248 Return:
249 249 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
250 250 fecha especificado, de lo contrario retorna False.
251 251
252 252 Excepciones:
253 253 Si el archivo no existe o no puede ser abierto
254 254 Si la cabecera no puede ser leida.
255 255
256 256 """
257 257
258 258 try:
259 259 fp = fp = h5py.File(filename,'r')
260 260 except IOError:
261 print "File %s can't be opened" %(filename)
262 return None
261 traceback.print_exc()
262 raise IOError, "The file %s can't be opened" %(filename)
263 263
264 264 grp = fp['Data']
265 265 timeAux = grp['time']
266 266 time0 = timeAux[:][0].astype(numpy.float) #Time Vector
267 267
268 268 fp.close()
269 269
270 270 if self.timezone == 'lt':
271 271 time0 -= 5*3600
272 272
273 273 boolTimer = numpy.logical_and(time0 >= startSeconds,time0 < endSeconds)
274 274
275 275 if not (numpy.any(boolTimer)):
276 276 return None
277 277
278 278 thisDatetime = datetime.datetime.utcfromtimestamp(time0[0])
279 279 return thisDatetime
280 280
281 281 def __checkPath(self):
282 282 if os.path.exists(self.path):
283 283 self.status = 1
284 284 else:
285 285 self.status = 0
286 286 print 'Path:%s does not exists'%self.path
287 287
288 288 return
289 289
290 290 def __setNextFileOffline(self):
291 291 idFile = self.fileIndex
292 292 idFile += 1
293 293
294 294 if not(idFile < len(self.filenameList)):
295 295 print "No more Files"
296 296 return 0
297 297
298 298 filename = self.filenameList[idFile]
299 299
300 300 filePointer = h5py.File(filename,'r')
301 301
302 302 self.flagIsNewFile = 1
303 303 self.fileIndex = idFile
304 304 self.filename = filename
305 305
306 306 self.fp = filePointer
307 307
308 308 print "Setting the file: %s"%self.filename
309 309
310 310 self.__readMetadata()
311 311 self.__setBlockList()
312 312 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
313 313 self.nRecords = self.fp['Data'].attrs['nRecords']
314 314 self.blockIndex = 0
315 315 return 1
316 316
317 317 def __setBlockList(self):
318 318 '''
319 319 self.fp
320 320 self.startDateTime
321 321 self.endDateTime
322 322
323 323 self.blockList
324 324 self.blocksPerFile
325 325
326 326 '''
327 327 filePointer = self.fp
328 328 secStart = self.secStart
329 329 secEnd = self.secEnd
330 330
331 331 grp = filePointer['Data']
332 332 timeVector = grp['time'].value.astype(numpy.float)[0]
333 333
334 334 if self.timezone == 'lt':
335 335 timeVector -= 5*3600
336 336
337 337 ind = numpy.where(numpy.logical_and(timeVector >= secStart , timeVector < secEnd))[0]
338 338
339 339 self.blockList = ind
340 340 self.blocksPerFile = len(ind)
341 341
342 342 return
343 343
344 344 def __readMetadata(self):
345 345 '''
346 346 self.pathMeta
347 347
348 348 self.listShapes
349 349 self.listMetaname
350 350 self.listMeta
351 351
352 352 '''
353 353
354 354 grp = self.fp['Data']
355 355 pathMeta = os.path.join(self.path, grp.attrs['metadata'])
356 356
357 357 if pathMeta == self.pathMeta:
358 358 return
359 359 else:
360 360 self.pathMeta = pathMeta
361 361
362 362 filePointer = h5py.File(self.pathMeta,'r')
363 363 groupPointer = filePointer['Metadata']
364 364
365 365 listMetaname = []
366 366 listMetadata = []
367 367 for item in groupPointer.items():
368 368 name = item[0]
369 369
370 370 if name=='array dimensions':
371 371 table = groupPointer[name][:]
372 372 listShapes = {}
373 373 for shapes in table:
374 374 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4]])
375 375 else:
376 376 data = groupPointer[name].value
377 377 listMetaname.append(name)
378 378 listMetadata.append(data)
379 379
380 380 if name=='type':
381 381 self.__initDataOut(data)
382 382
383 383 filePointer.close()
384 384
385 385 self.listShapes = listShapes
386 386 self.listMetaname = listMetaname
387 387 self.listMeta = listMetadata
388 388
389 389 return
390 390
391 391 def __readData(self):
392 392 grp = self.fp['Data']
393 393 listdataname = []
394 394 listdata = []
395 395
396 396 for item in grp.items():
397 397 name = item[0]
398 398
399 399 if name == 'time':
400 400 listdataname.append('utctime')
401 401 timeAux = grp[name].value.astype(numpy.float)[0]
402 402 listdata.append(timeAux)
403 403 continue
404 404
405 405 listdataname.append(name)
406 406 array = self.__setDataArray(self.nRecords, grp[name],self.listShapes[name])
407 407 listdata.append(array)
408 408
409 409 self.listDataname = listdataname
410 410 self.listData = listdata
411 411 return
412 412
413 413 def __setDataArray(self, nRecords, dataset, shapes):
414 414
415 415 nChannels = shapes[0] #Dimension 0
416 416
417 417 nPoints = shapes[1] #Dimension 1, number of Points or Parameters
418 418
419 419 nSamples = shapes[2] #Dimension 2, number of samples or ranges
420 420
421 421 mode = shapes[3]
422 422
423 423 # if nPoints>1:
424 424 # arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
425 425 # else:
426 426 # arrayData = numpy.zeros((nRecords,nChannels,nSamples))
427 427 #
428 428 # chn = 'channel'
429 429 #
430 430 # for i in range(nChannels):
431 431 #
432 432 # data = dataset[chn + str(i)].value
433 433 #
434 434 # if nPoints>1:
435 435 # data = numpy.rollaxis(data,2)
436 436 #
437 437 # arrayData[:,i,:] = data
438 438
439 439 arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
440 440 doSqueeze = False
441 441 if mode == 0:
442 442 strds = 'channel'
443 443 nDatas = nChannels
444 444 newShapes = (nRecords,nPoints,nSamples)
445 445 if nPoints == 1:
446 446 doSqueeze = True
447 447 axisSqueeze = 2
448 448 else:
449 449 strds = 'param'
450 450 nDatas = nPoints
451 451 newShapes = (nRecords,nChannels,nSamples)
452 452 if nChannels == 1:
453 453 doSqueeze = True
454 454 axisSqueeze = 1
455 455
456 456 for i in range(nDatas):
457 457
458 458 data = dataset[strds + str(i)].value
459 459 data = data.reshape(newShapes)
460 460
461 461 if mode == 0:
462 462 arrayData[:,i,:,:] = data
463 463 else:
464 464 arrayData[:,:,i,:] = data
465 465
466 466 if doSqueeze:
467 467 arrayData = numpy.squeeze(arrayData, axis=axisSqueeze)
468 468
469 469 return arrayData
470 470
471 471 def __initDataOut(self, type):
472 472
473 473 # if type =='Parameters':
474 474 # self.dataOut = Parameters()
475 475 # elif type =='Spectra':
476 476 # self.dataOut = Spectra()
477 477 # elif type =='Voltage':
478 478 # self.dataOut = Voltage()
479 479 # elif type =='Correlation':
480 480 # self.dataOut = Correlation()
481 481
482 482 return
483 483
484 484 def __setDataOut(self):
485 485 listMeta = self.listMeta
486 486 listMetaname = self.listMetaname
487 487 listDataname = self.listDataname
488 488 listData = self.listData
489 489
490 490 blockIndex = self.blockIndex
491 491 blockList = self.blockList
492 492
493 493 for i in range(len(listMeta)):
494 494 setattr(self.dataOut,listMetaname[i],listMeta[i])
495 495
496 496 for j in range(len(listData)):
497 497 if listDataname[j]=='utctime':
498 498 # setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex]])
499 499 setattr(self.dataOut,'utctimeInit',listData[j][blockList[blockIndex]])
500 500 continue
501 501
502 502 setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex],:])
503 503
504 504 return self.dataOut.data_param
505 505
506 506 def getData(self):
507 507
508 508 # if self.flagNoMoreFiles:
509 509 # self.dataOut.flagNoData = True
510 510 # print 'Process finished'
511 511 # return 0
512 512 #
513 513 if self.blockIndex==self.blocksPerFile:
514 514 if not( self.__setNextFileOffline() ):
515 515 self.dataOut.flagNoData = True
516 516 return 0
517 517
518 518 #
519 519 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
520 520 # self.dataOut.flagNoData = True
521 521 # return 0
522 522
523 523 self.__readData()
524 524 self.__setDataOut()
525 525 self.dataOut.flagNoData = False
526 526
527 527 self.blockIndex += 1
528 528
529 529 return
530 530
531 531 def run(self, **kwargs):
532 532
533 533 if not(self.isConfig):
534 534 self.setup(**kwargs)
535 535 # self.setObjProperties()
536 536 self.isConfig = True
537 537
538 538 self.getData()
539 539
540 540 return
541 541
542 542 class HDF5Writer(Operation):
543 543
544 544 ext = ".hdf5"
545 545
546 546 optchar = "D"
547 547
548 548 metaoptchar = "M"
549 549
550 550 metaFile = None
551 551
552 552 filename = None
553 553
554 554 path = None
555 555
556 556 setFile = None
557 557
558 558 fp = None
559 559
560 560 grp = None
561 561
562 562 ds = None
563 563
564 564 firsttime = True
565 565
566 566 #Configurations
567 567
568 568 blocksPerFile = None
569 569
570 570 blockIndex = None
571 571
572 572 dataOut = None
573 573
574 574 #Data Arrays
575 575
576 576 dataList = None
577 577
578 578 metadataList = None
579 579
580 580 arrayDim = None
581 581
582 582 tableDim = None
583 583
584 584 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
585 585
586 586 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
587 587
588 588 mode = None
589 589
590 590 nDatas = None #Number of datasets to be stored per array
591 591
592 592 nDims = None #Number Dimensions in each dataset
593 593
594 594 nDimsForDs = None
595 595
596 currentDay = None
597
596 598 def __init__(self):
597 599
598 600 Operation.__init__(self)
599 601 self.isConfig = False
600 602 return
601 603
602
603 604 def setup(self, dataOut, **kwargs):
604 605
605 606 self.path = kwargs['path']
606 607
607 608 if kwargs.has_key('ext'):
608 609 self.ext = kwargs['ext']
609 610
610 611 if kwargs.has_key('blocksPerFile'):
611 612 self.blocksPerFile = kwargs['blocksPerFile']
612 613 else:
613 614 self.blocksPerFile = 10
614 615
615 616 self.metadataList = kwargs['metadataList']
616 617
617 618 self.dataList = kwargs['dataList']
618 619
619 620 self.dataOut = dataOut
620 621
621 622 if kwargs.has_key('mode'):
622 623 mode = kwargs['mode']
623 624
624 625 if type(mode) == int:
625 626 mode = numpy.zeros(len(self.dataList)) + mode
626 627 else:
627 628 mode = numpy.zeros(len(self.dataList))
628 629
629 630 self.mode = mode
630 631
631 632 arrayDim = numpy.zeros((len(self.dataList),5))
632 633
633 634 #Table dimensions
634 635
635 636 dtype0 = self.dtype
636 637
637 638 tableList = []
638 639
639 640 for i in range(len(self.dataList)):
640 641
641 642 dataAux = getattr(self.dataOut, self.dataList[i])
642 643
643 644 if type(dataAux)==float or type(dataAux)==int:
644 645 arrayDim[i,0] = 1
645 646 else:
647
648 if dataAux == None:
649 return 0
650
646 651 arrayDim0 = dataAux.shape
647 652 arrayDim[i,0] = len(arrayDim0)
648 653 arrayDim[i,4] = mode[i]
649 654
650 655 if len(arrayDim0) == 3:
651 656 arrayDim[i,1:-1] = numpy.array(arrayDim0)
652 657 elif len(arrayDim0) == 2:
653 658 arrayDim[i,2:-1] = numpy.array(arrayDim0) #nHeights
654 659 elif len(arrayDim0) == 1:
655 660 arrayDim[i,3] = arrayDim0
656 661 elif len(arrayDim0) == 0:
657 662 arrayDim[i,0] = 1
658 663 arrayDim[i,3] = 1
659 664
660 665 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
661 666 tableList.append(table)
662 667
663 668 self.arrayDim = arrayDim
664 669 self.tableDim = numpy.array(tableList, dtype = dtype0)
665 670 self.blockIndex = 0
666 671
667 return
672 timeTuple = time.localtime(dataOut.utctime)
673 self.currentDay = timeTuple.tm_yday
674 return 1
668 675
669 676 def putMetadata(self):
670 677
671 678 fp = self.createMetadataFile()
672 679 self.writeMetadata(fp)
673 680 fp.close()
674 681 return
675 682
676 683 def createMetadataFile(self):
677 684 ext = self.ext
678 685 path = self.path
679 686 setFile = self.setFile
680 687
681 688 timeTuple = time.localtime(self.dataOut.utctime)
682 689
683 690 subfolder = ''
684 691 fullpath = os.path.join( path, subfolder )
685 692
686 693 if not( os.path.exists(fullpath) ):
687 694 os.mkdir(fullpath)
688 695 setFile = -1 #inicializo mi contador de seteo
689 696
690 697 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
691 698 fullpath = os.path.join( path, subfolder )
692 699
693 700 if not( os.path.exists(fullpath) ):
694 701 os.mkdir(fullpath)
695 702 setFile = -1 #inicializo mi contador de seteo
696 703
697 704 else:
698 705 filesList = os.listdir( fullpath )
699 706 filesList = sorted( filesList, key=str.lower )
700 707 if len( filesList ) > 0:
701 708 filesList = [k for k in filesList if 'M' in k]
702 709 filen = filesList[-1]
703 710 # el filename debera tener el siguiente formato
704 711 # 0 1234 567 89A BCDE (hex)
705 712 # x YYYY DDD SSS .ext
706 713 if isNumber( filen[8:11] ):
707 714 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
708 715 else:
709 716 setFile = -1
710 717 else:
711 718 setFile = -1 #inicializo mi contador de seteo
712 719
713 720 setFile += 1
714 721
715 722 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
716 723 timeTuple.tm_year,
717 724 timeTuple.tm_yday,
718 725 setFile,
719 726 ext )
720 727
721 728 filename = os.path.join( path, subfolder, file )
722 729 self.metaFile = file
723 730 #Setting HDF5 File
724 731 fp = h5py.File(filename,'w')
725 732
726 733 return fp
727 734
728 735 def writeMetadata(self, fp):
729 736
730 737 grp = fp.create_group("Metadata")
731 738 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
732 739
733 740 for i in range(len(self.metadataList)):
734 741 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
735 742 return
736 743
744 def dateFlag(self):
745
746 timeTuple = time.localtime(self.dataOut.utctime)
747 dataDay = timeTuple.tm_yday
748
749 if dataDay == self.currentDay:
750 return False
751
752 self.currentDay = dataDay
753 return True
754
737 755 def setNextFile(self):
738 756
739 757 ext = self.ext
740 758 path = self.path
741 759 setFile = self.setFile
742 760 mode = self.mode
743 761
744 762 timeTuple = time.localtime(self.dataOut.utctime)
745 763 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
746 764
747 765 fullpath = os.path.join( path, subfolder )
748 766
749 767 if os.path.exists(fullpath):
750 768 filesList = os.listdir( fullpath )
751 769 filesList = [k for k in filesList if 'D' in k]
752 770 if len( filesList ) > 0:
753 771 filesList = sorted( filesList, key=str.lower )
754 772 filen = filesList[-1]
755 773 # el filename debera tener el siguiente formato
756 774 # 0 1234 567 89A BCDE (hex)
757 775 # x YYYY DDD SSS .ext
758 776 if isNumber( filen[8:11] ):
759 777 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
760 778 else:
761 779 setFile = -1
762 780 else:
763 781 setFile = -1 #inicializo mi contador de seteo
764
782 else:
783 os.mkdir(fullpath)
784 setFile = -1 #inicializo mi contador de seteo
785
765 786 setFile += 1
766 787
767 788 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
768 789 timeTuple.tm_year,
769 790 timeTuple.tm_yday,
770 791 setFile,
771 792 ext )
772 793
773 794 filename = os.path.join( path, subfolder, file )
774 795
775 796 #Setting HDF5 File
776 797 fp = h5py.File(filename,'w')
798
799 #writemetadata
800 self.writeMetadata(fp)
801
777 802 grp = fp.create_group("Data")
778 grp.attrs['metadata'] = self.metaFile
803 # grp.attrs['metadata'] = self.metaFile
779 804
780 805 # grp.attrs['blocksPerFile'] = 0
781 806
782 807 ds = []
783 808 data = []
784 809 nDimsForDs = []
785 810
786 811 nDatas = numpy.zeros(len(self.dataList))
787 812 nDims = self.arrayDim[:,0]
788 813
789 814 nDim1 = self.arrayDim[:,2]
790 815 nDim0 = self.arrayDim[:,3]
791 816
792 817 for i in range(len(self.dataList)):
793 818
794 819 if nDims[i]==1:
795 820 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
796 821 ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
797 822 ds.append(ds0)
798 823 data.append([])
799 824 nDimsForDs.append(nDims[i])
800 825 else:
801 826
802 827 if mode[i]==0:
803 828 strMode = "channel"
804 829 nDatas[i] = self.arrayDim[i,1]
805 830 else:
806 831 strMode = "param"
807 832 nDatas[i] = self.arrayDim[i,2]
808 833
809 834 if nDims[i]==2:
810 835 nDatas[i] = self.arrayDim[i,2]
811 836
812 837 grp0 = grp.create_group(self.dataList[i])
813 838
814 839 for j in range(int(nDatas[i])):
815 840 tableName = strMode + str(j)
816 841
817 842 if nDims[i] == 3:
818 843 ds0 = grp0.create_dataset(tableName, (nDim1[i],nDim0[i],1) , data = numpy.zeros((nDim1[i],nDim0[i],1)) ,maxshape=(None,nDim0[i],None), chunks=True)
819 844 else:
820 845 ds0 = grp0.create_dataset(tableName, (1,nDim0[i]), data = numpy.zeros((1,nDim0[i])) , maxshape=(None,nDim0[i]), chunks=True)
821 846
822 847 ds.append(ds0)
823 848 data.append([])
824 849 nDimsForDs.append(nDims[i])
825 850 self.nDatas = nDatas
826 851 self.nDims = nDims
827 852 self.nDimsForDs = nDimsForDs
828 853 #Saving variables
829 854 print 'Writing the file: %s'%filename
830 855 self.filename = filename
831 856 self.fp = fp
832 857 self.grp = grp
833 858 self.grp.attrs.modify('nRecords', 1)
834 859 self.ds = ds
835 860 self.data = data
836 861
837 862 self.setFile = setFile
838 863 self.firsttime = True
839 864 self.blockIndex = 0
840 865 return
841 866
842 867 def putData(self):
843 868
844 869 if not self.firsttime:
845 870 self.readBlock()
846 871
847 if self.blockIndex == self.blocksPerFile:
872 if self.blockIndex == self.blocksPerFile or self.dateFlag():
848 873
849 874 self.setNextFile()
850 875
851 876 self.setBlock()
852 877 self.writeBlock()
853 878
854 879 self.fp.flush()
855 880 self.fp.close()
856 881
857 882 return
858 883
859 884 def readBlock(self):
860 885
861 886 '''
862 887 data Array configured
863 888
864 889
865 890 self.data
866 891 '''
867 892 ds = self.ds
868 893 #Setting HDF5 File
869 894 fp = h5py.File(self.filename,'r+')
870 895 grp = fp["Data"]
871 896 ind = 0
872 897
873 898 # grp.attrs['blocksPerFile'] = 0
874 899 for i in range(len(self.dataList)):
875 900
876 901 if self.nDims[i]==1:
877 902 ds0 = grp[self.dataList[i]]
878 903 ds[ind] = ds0
879 904 ind += 1
880 905 else:
881 906 if self.mode[i]==0:
882 907 strMode = "channel"
883 908 else:
884 909 strMode = "param"
885 910
886 911 grp0 = grp[self.dataList[i]]
887 912
888 913 for j in range(int(self.nDatas[i])):
889 914 tableName = strMode + str(j)
890 915 ds0 = grp0[tableName]
891 916 ds[ind] = ds0
892 917 ind += 1
893 918
894 919
895 920 self.fp = fp
896 921 self.grp = grp
897 922 self.ds = ds
898 923
899 924 return
900 925
901
902 926 def setBlock(self):
903 927 '''
904 928 data Array configured
905 929
906 930
907 931 self.data
908 932 '''
909 933 #Creating Arrays
910 934 data = self.data
911 935 nDatas = self.nDatas
912 936 nDims = self.nDims
913 937 mode = self.mode
914 938 ind = 0
915 939
916 940 for i in range(len(self.dataList)):
917 941 dataAux = getattr(self.dataOut,self.dataList[i])
918 942
919 943 if nDims[i] == 1:
920 944 # data[ind] = numpy.array([str(dataAux)]).reshape((1,1))
921 945 data[ind] = dataAux
922 946 # if not self.firsttime:
923 947 # data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind]))
924 948 ind += 1
925 949 else:
926 950 for j in range(int(nDatas[i])):
927 951 if (mode[i] == 0) or (nDims[i] == 2): #In case division per channel or Dimensions is only 1
928 952 data[ind] = dataAux[j,:]
929 953 else:
930 954 data[ind] = dataAux[:,j,:]
931 955
932 956 # if nDims[i] == 3:
933 957 # data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1))
934 958
935 959 # if not self.firsttime:
936 960 # data[ind] = numpy.dstack((self.ds[ind][:], data[ind]))
937 961
938 962 # else:
939 963 # data[ind] = data[ind].reshape((1,data[ind].shape[0]))
940 964
941 965 # if not self.firsttime:
942 966 # data[ind] = numpy.vstack((self.ds[ind][:], data[ind]))
943 967 ind += 1
944 968
945 969 self.data = data
946 970 return
947 971
948 972 def writeBlock(self):
949 973 '''
950 974 Saves the block in the HDF5 file
951 975 '''
952 976 for i in range(len(self.ds)):
953 977 if self.firsttime:
954 978 # self.ds[i].resize(self.data[i].shape)
955 979 # self.ds[i][self.blockIndex,:] = self.data[i]
956 980 if type(self.data[i]) == numpy.ndarray:
957 981 nDims1 = len(self.ds[i].shape)
958 982
959 983 if nDims1 == 3:
960 984 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
961 985
962 986 self.ds[i].resize(self.data[i].shape)
963 987 self.ds[i][:] = self.data[i]
964 988 else:
965 989 if self.nDimsForDs[i] == 1:
966 990 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
967 991 self.ds[i][0,-1] = self.data[i]
968 992 elif self.nDimsForDs[i] == 2:
969 993 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
970 994 self.ds[i][self.blockIndex,:] = self.data[i]
971 995 elif self.nDimsForDs[i] == 3:
972 996
973 997 dataShape = self.data[i].shape
974 998 dsShape = self.ds[i].shape
975 999
976 1000 if dataShape[0]==dsShape[0]:
977 1001 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
978 1002 self.ds[i][:,:,-1] = self.data[i]
979 1003 else:
980 1004 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1],self.ds[i].shape[2]))
981 1005 self.ds[i][dsShape[0]:,:,0] = self.data[i]
982 # self.ds[i].append(self.data[i])
983 # self.fp.flush()
984 # if not self.firsttime:
985 # self.fp.root.Data._v_attrs.nRecords = self.blockIndex
986
987 # if self.firsttime:
988 # self.fp.close()
989 # self.readBlock2()
990 1006
991 1007 self.blockIndex += 1
992 1008 self.firsttime = False
993 1009 return
994 1010
995 1011 def run(self, dataOut, **kwargs):
1012
996 1013 if not(self.isConfig):
997 self.setup(dataOut, **kwargs)
1014 flagdata = self.setup(dataOut, **kwargs)
1015
1016 if not(flagdata):
1017 return
1018
998 1019 self.isConfig = True
999 self.putMetadata()
1020 # self.putMetadata()
1000 1021 self.setNextFile()
1001 1022
1002 1023 self.putData()
1003 1024 return
1004 1025
1026
General Comments 0
You need to be logged in to leave comments. Login now