##// END OF EJS Templates
schain-jc
RSS
-bug fix in HDF5 Writer Module
Julio Valdez -
r810:7e14525bff70
parent child
Show More
Show file before | Show file after | Hide comments
@@ -577,7 +577,9 class HDF5Writer(Operation):
577
577
578 metadataList = None
578 metadataList = None
579
579
580 arrayDim = None
580 # arrayDim = None
581
582 dsList = None #List of dictionaries
581
583
582 tableDim = None
584 tableDim = None
583
585
@@ -611,9 +613,7 class HDF5Writer(Operation):
611 self.blocksPerFile = 10
613 self.blocksPerFile = 10
612
614
613 self.metadataList = kwargs['metadataList']
615 self.metadataList = kwargs['metadataList']
614
615 self.dataList = kwargs['dataList']
616 self.dataList = kwargs['dataList']
616
617 self.dataOut = dataOut
617 self.dataOut = dataOut
618
618
619 if kwargs.has_key('mode'):
619 if kwargs.has_key('mode'):
@@ -629,57 +629,91 class HDF5Writer(Operation):
629 arrayDim = numpy.zeros((len(self.dataList),5))
629 arrayDim = numpy.zeros((len(self.dataList),5))
630
630
631 #Table dimensions
631 #Table dimensions
632
633 dtype0 = self.dtype
632 dtype0 = self.dtype
634
635 tableList = []
633 tableList = []
636
634
635 #Dictionary and list of tables
636 dsList = []
637
637 for i in range(len(self.dataList)):
638 for i in range(len(self.dataList)):
638
639 dsDict = {}
639 dataAux = getattr(self.dataOut, self.dataList[i])
640 dataAux = getattr(self.dataOut, self.dataList[i])
640
641 dsDict['variable'] = self.dataList[i]
641 #--------------------- Conditionals ------------------------
642 #--------------------- Conditionals ------------------------
642 #There is no data
643 #There is no data
643 if dataAux == None:
644 if dataAux == None:
644 return 0
645 return 0
645
646
646 #Not array, just a number
647 #Not array, just a number
647 if type(dataAux)==float or type(dataAux)==int:
648 #Mode 0
649 if type(dataAux)==float or type(dataAux)==int:
650 dsDict['mode'] = 0
651 dsDict['nDim'] = 0
648 arrayDim[i,0] = 1
652 arrayDim[i,0] = 1
649 mode[i] = 0
653 dsList.append(dsDict)
650
654
651 #Mode meteors
655 #Mode 2: meteors
652 elif mode[i] == 2:
656 elif mode[i] == 2:
657 # dsDict['nDim'] = 0
658 dsDict['dsName'] = 'table0'
659 dsDict['mode'] = 2 # Mode meteors
660 dsDict['shape'] = dataAux.shape[-1]
661 dsDict['nDim'] = 0
662
653 arrayDim[i,3] = dataAux.shape[-1]
663 arrayDim[i,3] = dataAux.shape[-1]
654 arrayDim[i,4] = mode[i] #Mode the data was stored
664 arrayDim[i,4] = mode[i] #Mode the data was stored
665
666 dsList.append(dsDict)
655
667
656 #All the rest
668 #Mode 1
657 else:
669 else:
658 arrayDim0 = dataAux.shape #Data dimensions
670 arrayDim0 = dataAux.shape #Data dimensions
659 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
671 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
660 arrayDim[i,4] = mode[i] #Mode the data was stored
672 arrayDim[i,4] = mode[i] #Mode the data was stored
661
673
674 strtable = 'table'
675 dsDict['mode'] = 1 # Mode parameters
676
662 # Three-dimension arrays
677 # Three-dimension arrays
663 if len(arrayDim0) == 3:
678 if len(arrayDim0) == 3:
664 arrayDim[i,1:-1] = numpy.array(arrayDim0)
679 arrayDim[i,1:-1] = numpy.array(arrayDim0)
665
680 nTables = int(arrayDim[i,2])
681 dsDict['dsNumber'] = nTables
682 dsDict['shape'] = arrayDim[i,2:4]
683 dsDict['nDim'] = 3
684
685 for j in range(nTables):
686 dsDict = dsDict.copy()
687 dsDict['dsName'] = strtable + str(j)
688 dsList.append(dsDict)
689
666 # Two-dimension arrays
690 # Two-dimension arrays
667 elif len(arrayDim0) == 2:
691 elif len(arrayDim0) == 2:
668 arrayDim[i,2:-1] = numpy.array(arrayDim0)
692 arrayDim[i,2:-1] = numpy.array(arrayDim0)
669
693 nTables = int(arrayDim[i,2])
694 dsDict['dsNumber'] = nTables
695 dsDict['shape'] = arrayDim[i,3]
696 dsDict['nDim'] = 2
697
698 for j in range(nTables):
699 dsDict = dsDict.copy()
700 dsDict['dsName'] = strtable + str(j)
701 dsList.append(dsDict)
702
670 # One-dimension arrays
703 # One-dimension arrays
671 elif len(arrayDim0) == 1:
704 elif len(arrayDim0) == 1:
672 arrayDim[i,3] = arrayDim0
705 arrayDim[i,3] = arrayDim0[0]
673
706 dsDict['shape'] = arrayDim0[0]
674 # No array, just a number
707 dsDict['dsNumber'] = 1
675 elif len(arrayDim0) == 0:
708 dsDict['dsName'] = strtable + str(0)
676 arrayDim[i,0] = 1
709 dsDict['nDim'] = 1
677 arrayDim[i,3] = 1
710 dsList.append(dsDict)
678
711
679 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
712 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
680 tableList.append(table)
713 tableList.append(table)
681
714
682 self.arrayDim = arrayDim
715 # self.arrayDim = arrayDim
716 self.dsList = dsList
683 self.tableDim = numpy.array(tableList, dtype = dtype0)
717 self.tableDim = numpy.array(tableList, dtype = dtype0)
684 self.blockIndex = 0
718 self.blockIndex = 0
685
719
@@ -809,6 +843,117 class HDF5Writer(Operation):
809
843
810 #Setting HDF5 File
844 #Setting HDF5 File
811 fp = h5py.File(filename,'w')
845 fp = h5py.File(filename,'w')
846 #write metadata
847 self.writeMetadata(fp)
848 #Write data
849 grp = fp.create_group("Data")
850 # grp.attrs['metadata'] = self.metaFile
851
852 # grp.attrs['blocksPerFile'] = 0
853 ds = []
854 data = []
855 dsList = self.dsList
856 i = 0
857 while i < len(dsList):
858 dsInfo = dsList[i]
859 #One-dimension data
860 if dsInfo['mode'] == 0:
861 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
862 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
863 ds.append(ds0)
864 data.append([])
865 i += 1
866 continue
867 # nDimsForDs.append(nDims[i])
868
869 elif dsInfo['mode'] == 2:
870 grp0 = grp.create_group(dsInfo['variable'])
871 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
872 ds.append(ds0)
873 data.append([])
874 i += 1
875 continue
876
877 elif dsInfo['mode'] == 1:
878 grp0 = grp.create_group(dsInfo['variable'])
879
880 for j in range(dsInfo['dsNumber']):
881 dsInfo = dsList[i]
882 tableName = dsInfo['dsName']
883 shape = dsInfo['shape']
884
885 if dsInfo['nDim'] == 3:
886 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
887 else:
888 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
889
890 ds.append(ds0)
891 data.append([])
892 i += 1
893 # nDimsForDs.append(nDims[i])
894
895 fp.flush()
896 fp.close()
897
898 # self.nDatas = nDatas
899 # self.nDims = nDims
900 # self.nDimsForDs = nDimsForDs
901 #Saving variables
902 print 'Writing the file: %s'%filename
903 self.filename = filename
904 # self.fp = fp
905 # self.grp = grp
906 # self.grp.attrs.modify('nRecords', 1)
907 self.ds = ds
908 self.data = data
909 # self.setFile = setFile
910 self.firsttime = True
911 self.blockIndex = 0
912 return
913
914 def setNextFile1(self):
915
916 ext = self.ext
917 path = self.path
918 setFile = self.setFile
919 mode = self.mode
920
921 timeTuple = time.localtime(self.dataOut.utctime)
922 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
923
924 fullpath = os.path.join( path, subfolder )
925
926 if os.path.exists(fullpath):
927 filesList = os.listdir( fullpath )
928 filesList = [k for k in filesList if 'D' in k]
929 if len( filesList ) > 0:
930 filesList = sorted( filesList, key=str.lower )
931 filen = filesList[-1]
932 # el filename debera tener el siguiente formato
933 # 0 1234 567 89A BCDE (hex)
934 # x YYYY DDD SSS .ext
935 if isNumber( filen[8:11] ):
936 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
937 else:
938 setFile = -1
939 else:
940 setFile = -1 #inicializo mi contador de seteo
941 else:
942 os.mkdir(fullpath)
943 setFile = -1 #inicializo mi contador de seteo
944
945 setFile += 1
946
947 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
948 timeTuple.tm_year,
949 timeTuple.tm_yday,
950 setFile,
951 ext )
952
953 filename = os.path.join( path, subfolder, file )
954
955 #Setting HDF5 File
956 fp = h5py.File(filename,'w')
812
957
813 #writemetadata
958 #writemetadata
814 self.writeMetadata(fp)
959 self.writeMetadata(fp)
@@ -914,6 +1059,7 class HDF5Writer(Operation):
914
1059
915 self.data
1060 self.data
916 '''
1061 '''
1062 dsList = self.dsList
917 ds = self.ds
1063 ds = self.ds
918 #Setting HDF5 File
1064 #Setting HDF5 File
919 fp = h5py.File(self.filename,'r+')
1065 fp = h5py.File(self.filename,'r+')
@@ -921,25 +1067,20 class HDF5Writer(Operation):
921 ind = 0
1067 ind = 0
922
1068
923 # grp.attrs['blocksPerFile'] = 0
1069 # grp.attrs['blocksPerFile'] = 0
924 for i in range(len(self.dataList)):
1070 while ind < len(dsList):
1071 dsInfo = dsList[ind]
925
1072
926 if self.nDims[i]==1:
1073 if dsInfo['mode'] == 0:
927 ds0 = grp[self.dataList[i]]
1074 ds0 = grp[dsInfo['variable']]
928 ds[ind] = ds0
1075 ds[ind] = ds0
929 ind += 1
1076 ind += 1
930 else:
1077 else:
931 # if self.mode[i] == 0:
932 # strMode = "channel"
933 if self.mode[i] == 1:
934 strMode = "param"
935 elif self.mode[i] == 2:
936 strMode = "table"
937
1078
938 grp0 = grp[self.dataList[i]]
1079 grp0 = grp[dsInfo['variable']]
939
1080
940 for j in range(int(self.nDatas[i])):
1081 for j in range(dsInfo['dsNumber']):
941 tableName = strMode + str(j)
1082 dsInfo = dsList[ind]
942 ds0 = grp0[tableName]
1083 ds0 = grp0[dsInfo['dsName']]
943 ds[ind] = ds0
1084 ds[ind] = ds0
944 ind += 1
1085 ind += 1
945
1086
@@ -957,33 +1098,32 class HDF5Writer(Operation):
957 self.data
1098 self.data
958 '''
1099 '''
959 #Creating Arrays
1100 #Creating Arrays
1101 dsList = self.dsList
960 data = self.data
1102 data = self.data
961 nDatas = self.nDatas
962 nDims = self.nDims
963 mode = self.mode
964 ind = 0
1103 ind = 0
965
1104
966 for i in range(len(self.dataList)):
1105 while ind < len(dsList):
967 dataAux = getattr(self.dataOut,self.dataList[i])
1106 dsInfo = dsList[ind]
1107 dataAux = getattr(self.dataOut, dsInfo['variable'])
968
1108
969 if nDims[i] == 1 or mode[i] == 2:
1109 mode = dsInfo['mode']
1110 nDim = dsInfo['nDim']
1111
1112 if mode == 0 or mode == 2 or nDim == 1:
970 data[ind] = dataAux
1113 data[ind] = dataAux
971 ind += 1
1114 ind += 1
972
1115 # elif nDim == 1:
973 elif nDims[i] == 2:
1116 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
974 for j in range(int(nDatas[i])):
1117 # ind += 1
1118 elif nDim == 2:
1119 for j in range(dsInfo['dsNumber']):
975 data[ind] = dataAux[j,:]
1120 data[ind] = dataAux[j,:]
976 ind += 1
1121 ind += 1
977
1122 elif nDim == 3:
978 elif nDims[i] == 3:
1123 for j in range(dsInfo['dsNumber']):
979 for j in range(int(nDatas[i])):
980 # Extinct mode 0
981 # if (mode[i] == 0):
982 # data[ind] = dataAux[j,:,:]
983 # else:
984 data[ind] = dataAux[:,j,:]
1124 data[ind] = dataAux[:,j,:]
985 ind += 1
1125 ind += 1
986
1126
987 self.data = data
1127 self.data = data
988 return
1128 return
989
1129
@@ -991,40 +1131,47 class HDF5Writer(Operation):
991 '''
1131 '''
992 Saves the block in the HDF5 file
1132 Saves the block in the HDF5 file
993 '''
1133 '''
1134 dsList = self.dsList
1135
994 for i in range(len(self.ds)):
1136 for i in range(len(self.ds)):
1137 dsInfo = dsList[i]
1138 nDim = dsInfo['nDim']
1139 mode = dsInfo['mode']
995
1140
996 # First time
1141 # First time
997 if self.firsttime:
1142 if self.firsttime:
998 # self.ds[i].resize(self.data[i].shape)
1143 # self.ds[i].resize(self.data[i].shape)
999 # self.ds[i][self.blockIndex,:] = self.data[i]
1144 # self.ds[i][self.blockIndex,:] = self.data[i]
1000 if type(self.data[i]) == numpy.ndarray:
1145 if type(self.data[i]) == numpy.ndarray:
1001 nDims1 = len(self.ds[i].shape)
1002
1146
1003 if nDims1 == 3:
1147 if nDim == 3:
1004 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1148 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1005
1149 self.ds[i].resize(self.data[i].shape)
1006 self.ds[i].resize(self.data[i].shape)
1007
1150
1008 self.ds[i][:] = self.data[i]
1151 self.ds[i][:] = self.data[i]
1009 else:
1152 else:
1010
1153
1011 # From second time
1154 # From second time
1012 # Meteors!
1155 # Meteors!
1013 if self.mode[i] == 2:
1156 if mode == 2:
1014 dataShape = self.data[i].shape
1157 dataShape = self.data[i].shape
1015 dsShape = self.ds[i].shape
1158 dsShape = self.ds[i].shape
1016 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1159 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1017 self.ds[i][dsShape[0]:,:] = self.data[i]
1160 self.ds[i][dsShape[0]:,:] = self.data[i]
1018 # One dimension
1161 # No dimension
1019 elif self.nDimsForDs[i] == 1:
1162 elif mode == 0:
1020 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1163 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1021 self.ds[i][0,-1] = self.data[i]
1164 self.ds[i][0,-1] = self.data[i]
1165 # One dimension
1166 elif nDim == 1:
1167 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1168 self.ds[i][-1,:] = self.data[i]
1022 # Two dimension
1169 # Two dimension
1023 elif self.nDimsForDs[i] == 2:
1170 elif nDim == 2:
1024 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1171 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1025 self.ds[i][self.blockIndex,:] = self.data[i]
1172 self.ds[i][self.blockIndex,:] = self.data[i]
1026 # Three dimensions
1173 # Three dimensions
1027 elif self.nDimsForDs[i] == 3:
1174 elif nDim == 3:
1028 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1175 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1029 self.ds[i][:,:,-1] = self.data[i]
1176 self.ds[i][:,:,-1] = self.data[i]
1030
1177
General Comments 0
You need to be logged in to leave comments. Login now