##// END OF EJS Templates
schain-jc
RSS
-bug fix in HDF5 Writer Module
Julio Valdez -
r810:7e14525bff70
parent child
Show More
Show file before | Show file after | Hide comments
@@ -577,7 +577,9 class HDF5Writer(Operation):
577 577
578 578 metadataList = None
579 579
580 arrayDim = None
580 # arrayDim = None
581
582 dsList = None #List of dictionaries
581 583
582 584 tableDim = None
583 585
@@ -611,9 +613,7 class HDF5Writer(Operation):
611 613 self.blocksPerFile = 10
612 614
613 615 self.metadataList = kwargs['metadataList']
614
615 616 self.dataList = kwargs['dataList']
616
617 617 self.dataOut = dataOut
618 618
619 619 if kwargs.has_key('mode'):
@@ -629,57 +629,91 class HDF5Writer(Operation):
629 629 arrayDim = numpy.zeros((len(self.dataList),5))
630 630
631 631 #Table dimensions
632
633 632 dtype0 = self.dtype
634
635 633 tableList = []
636 634
635 #Dictionary and list of tables
636 dsList = []
637
637 638 for i in range(len(self.dataList)):
638
639 dsDict = {}
639 640 dataAux = getattr(self.dataOut, self.dataList[i])
640
641 dsDict['variable'] = self.dataList[i]
641 642 #--------------------- Conditionals ------------------------
642 643 #There is no data
643 644 if dataAux == None:
644 645 return 0
645 646
646 647 #Not array, just a number
647 if type(dataAux)==float or type(dataAux)==int:
648 #Mode 0
649 if type(dataAux)==float or type(dataAux)==int:
650 dsDict['mode'] = 0
651 dsDict['nDim'] = 0
648 652 arrayDim[i,0] = 1
649 mode[i] = 0
650
651 #Mode meteors
652 elif mode[i] == 2:
653 dsList.append(dsDict)
654
655 #Mode 2: meteors
656 elif mode[i] == 2:
657 # dsDict['nDim'] = 0
658 dsDict['dsName'] = 'table0'
659 dsDict['mode'] = 2 # Mode meteors
660 dsDict['shape'] = dataAux.shape[-1]
661 dsDict['nDim'] = 0
662
653 663 arrayDim[i,3] = dataAux.shape[-1]
654 664 arrayDim[i,4] = mode[i] #Mode the data was stored
665
666 dsList.append(dsDict)
655 667
656 #All the rest
668 #Mode 1
657 669 else:
658 670 arrayDim0 = dataAux.shape #Data dimensions
659 671 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
660 672 arrayDim[i,4] = mode[i] #Mode the data was stored
661 673
674 strtable = 'table'
675 dsDict['mode'] = 1 # Mode parameters
676
662 677 # Three-dimension arrays
663 678 if len(arrayDim0) == 3:
664 679 arrayDim[i,1:-1] = numpy.array(arrayDim0)
665
680 nTables = int(arrayDim[i,2])
681 dsDict['dsNumber'] = nTables
682 dsDict['shape'] = arrayDim[i,2:4]
683 dsDict['nDim'] = 3
684
685 for j in range(nTables):
686 dsDict = dsDict.copy()
687 dsDict['dsName'] = strtable + str(j)
688 dsList.append(dsDict)
689
666 690 # Two-dimension arrays
667 691 elif len(arrayDim0) == 2:
668 692 arrayDim[i,2:-1] = numpy.array(arrayDim0)
669
693 nTables = int(arrayDim[i,2])
694 dsDict['dsNumber'] = nTables
695 dsDict['shape'] = arrayDim[i,3]
696 dsDict['nDim'] = 2
697
698 for j in range(nTables):
699 dsDict = dsDict.copy()
700 dsDict['dsName'] = strtable + str(j)
701 dsList.append(dsDict)
702
670 703 # One-dimension arrays
671 704 elif len(arrayDim0) == 1:
672 arrayDim[i,3] = arrayDim0
673
674 # No array, just a number
675 elif len(arrayDim0) == 0:
676 arrayDim[i,0] = 1
677 arrayDim[i,3] = 1
705 arrayDim[i,3] = arrayDim0[0]
706 dsDict['shape'] = arrayDim0[0]
707 dsDict['dsNumber'] = 1
708 dsDict['dsName'] = strtable + str(0)
709 dsDict['nDim'] = 1
710 dsList.append(dsDict)
678 711
679 712 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
680 713 tableList.append(table)
681 714
682 self.arrayDim = arrayDim
715 # self.arrayDim = arrayDim
716 self.dsList = dsList
683 717 self.tableDim = numpy.array(tableList, dtype = dtype0)
684 718 self.blockIndex = 0
685 719
@@ -809,6 +843,117 class HDF5Writer(Operation):
809 843
810 844 #Setting HDF5 File
811 845 fp = h5py.File(filename,'w')
846 #write metadata
847 self.writeMetadata(fp)
848 #Write data
849 grp = fp.create_group("Data")
850 # grp.attrs['metadata'] = self.metaFile
851
852 # grp.attrs['blocksPerFile'] = 0
853 ds = []
854 data = []
855 dsList = self.dsList
856 i = 0
857 while i < len(dsList):
858 dsInfo = dsList[i]
859 #One-dimension data
860 if dsInfo['mode'] == 0:
861 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
862 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
863 ds.append(ds0)
864 data.append([])
865 i += 1
866 continue
867 # nDimsForDs.append(nDims[i])
868
869 elif dsInfo['mode'] == 2:
870 grp0 = grp.create_group(dsInfo['variable'])
871 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
872 ds.append(ds0)
873 data.append([])
874 i += 1
875 continue
876
877 elif dsInfo['mode'] == 1:
878 grp0 = grp.create_group(dsInfo['variable'])
879
880 for j in range(dsInfo['dsNumber']):
881 dsInfo = dsList[i]
882 tableName = dsInfo['dsName']
883 shape = dsInfo['shape']
884
885 if dsInfo['nDim'] == 3:
886 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
887 else:
888 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
889
890 ds.append(ds0)
891 data.append([])
892 i += 1
893 # nDimsForDs.append(nDims[i])
894
895 fp.flush()
896 fp.close()
897
898 # self.nDatas = nDatas
899 # self.nDims = nDims
900 # self.nDimsForDs = nDimsForDs
901 #Saving variables
902 print 'Writing the file: %s'%filename
903 self.filename = filename
904 # self.fp = fp
905 # self.grp = grp
906 # self.grp.attrs.modify('nRecords', 1)
907 self.ds = ds
908 self.data = data
909 # self.setFile = setFile
910 self.firsttime = True
911 self.blockIndex = 0
912 return
913
914 def setNextFile1(self):
915
916 ext = self.ext
917 path = self.path
918 setFile = self.setFile
919 mode = self.mode
920
921 timeTuple = time.localtime(self.dataOut.utctime)
922 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
923
924 fullpath = os.path.join( path, subfolder )
925
926 if os.path.exists(fullpath):
927 filesList = os.listdir( fullpath )
928 filesList = [k for k in filesList if 'D' in k]
929 if len( filesList ) > 0:
930 filesList = sorted( filesList, key=str.lower )
931 filen = filesList[-1]
932 # el filename debera tener el siguiente formato
933 # 0 1234 567 89A BCDE (hex)
934 # x YYYY DDD SSS .ext
935 if isNumber( filen[8:11] ):
936 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
937 else:
938 setFile = -1
939 else:
940 setFile = -1 #inicializo mi contador de seteo
941 else:
942 os.mkdir(fullpath)
943 setFile = -1 #inicializo mi contador de seteo
944
945 setFile += 1
946
947 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
948 timeTuple.tm_year,
949 timeTuple.tm_yday,
950 setFile,
951 ext )
952
953 filename = os.path.join( path, subfolder, file )
954
955 #Setting HDF5 File
956 fp = h5py.File(filename,'w')
812 957
813 958 #writemetadata
814 959 self.writeMetadata(fp)
@@ -914,6 +1059,7 class HDF5Writer(Operation):
914 1059
915 1060 self.data
916 1061 '''
1062 dsList = self.dsList
917 1063 ds = self.ds
918 1064 #Setting HDF5 File
919 1065 fp = h5py.File(self.filename,'r+')
@@ -921,25 +1067,20 class HDF5Writer(Operation):
921 1067 ind = 0
922 1068
923 1069 # grp.attrs['blocksPerFile'] = 0
924 for i in range(len(self.dataList)):
1070 while ind < len(dsList):
1071 dsInfo = dsList[ind]
925 1072
926 if self.nDims[i]==1:
927 ds0 = grp[self.dataList[i]]
1073 if dsInfo['mode'] == 0:
1074 ds0 = grp[dsInfo['variable']]
928 1075 ds[ind] = ds0
929 1076 ind += 1
930 1077 else:
931 # if self.mode[i] == 0:
932 # strMode = "channel"
933 if self.mode[i] == 1:
934 strMode = "param"
935 elif self.mode[i] == 2:
936 strMode = "table"
937 1078
938 grp0 = grp[self.dataList[i]]
1079 grp0 = grp[dsInfo['variable']]
939 1080
940 for j in range(int(self.nDatas[i])):
941 tableName = strMode + str(j)
942 ds0 = grp0[tableName]
1081 for j in range(dsInfo['dsNumber']):
1082 dsInfo = dsList[ind]
1083 ds0 = grp0[dsInfo['dsName']]
943 1084 ds[ind] = ds0
944 1085 ind += 1
945 1086
@@ -957,33 +1098,32 class HDF5Writer(Operation):
957 1098 self.data
958 1099 '''
959 1100 #Creating Arrays
1101 dsList = self.dsList
960 1102 data = self.data
961 nDatas = self.nDatas
962 nDims = self.nDims
963 mode = self.mode
964 1103 ind = 0
965 1104
966 for i in range(len(self.dataList)):
967 dataAux = getattr(self.dataOut,self.dataList[i])
1105 while ind < len(dsList):
1106 dsInfo = dsList[ind]
1107 dataAux = getattr(self.dataOut, dsInfo['variable'])
968 1108
969 if nDims[i] == 1 or mode[i] == 2:
1109 mode = dsInfo['mode']
1110 nDim = dsInfo['nDim']
1111
1112 if mode == 0 or mode == 2 or nDim == 1:
970 1113 data[ind] = dataAux
971 ind += 1
972
973 elif nDims[i] == 2:
974 for j in range(int(nDatas[i])):
1114 ind += 1
1115 # elif nDim == 1:
1116 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1117 # ind += 1
1118 elif nDim == 2:
1119 for j in range(dsInfo['dsNumber']):
975 1120 data[ind] = dataAux[j,:]
976 1121 ind += 1
977
978 elif nDims[i] == 3:
979 for j in range(int(nDatas[i])):
980 # Extinct mode 0
981 # if (mode[i] == 0):
982 # data[ind] = dataAux[j,:,:]
983 # else:
1122 elif nDim == 3:
1123 for j in range(dsInfo['dsNumber']):
984 1124 data[ind] = dataAux[:,j,:]
985 1125 ind += 1
986
1126
987 1127 self.data = data
988 1128 return
989 1129
@@ -991,40 +1131,47 class HDF5Writer(Operation):
991 1131 '''
992 1132 Saves the block in the HDF5 file
993 1133 '''
1134 dsList = self.dsList
1135
994 1136 for i in range(len(self.ds)):
1137 dsInfo = dsList[i]
1138 nDim = dsInfo['nDim']
1139 mode = dsInfo['mode']
995 1140
996 1141 # First time
997 1142 if self.firsttime:
998 1143 # self.ds[i].resize(self.data[i].shape)
999 1144 # self.ds[i][self.blockIndex,:] = self.data[i]
1000 1145 if type(self.data[i]) == numpy.ndarray:
1001 nDims1 = len(self.ds[i].shape)
1002 1146
1003 if nDims1 == 3:
1147 if nDim == 3:
1004 1148 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1005
1006 self.ds[i].resize(self.data[i].shape)
1149 self.ds[i].resize(self.data[i].shape)
1007 1150
1008 1151 self.ds[i][:] = self.data[i]
1009 1152 else:
1010 1153
1011 1154 # From second time
1012 1155 # Meteors!
1013 if self.mode[i] == 2:
1156 if mode == 2:
1014 1157 dataShape = self.data[i].shape
1015 1158 dsShape = self.ds[i].shape
1016 1159 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1017 1160 self.ds[i][dsShape[0]:,:] = self.data[i]
1018 # One dimension
1019 elif self.nDimsForDs[i] == 1:
1161 # No dimension
1162 elif mode == 0:
1020 1163 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1021 1164 self.ds[i][0,-1] = self.data[i]
1165 # One dimension
1166 elif nDim == 1:
1167 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1168 self.ds[i][-1,:] = self.data[i]
1022 1169 # Two dimension
1023 elif self.nDimsForDs[i] == 2:
1170 elif nDim == 2:
1024 1171 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1025 1172 self.ds[i][self.blockIndex,:] = self.data[i]
1026 1173 # Three dimensions
1027 elif self.nDimsForDs[i] == 3:
1174 elif nDim == 3:
1028 1175 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1029 1176 self.ds[i][:,:,-1] = self.data[i]
1030 1177
General Comments 0
You need to be logged in to leave comments. Login now