@@ -577,7 +577,9 class HDF5Writer(Operation): | |||
|
577 | 577 | |
|
578 | 578 | metadataList = None |
|
579 | 579 | |
|
580 | arrayDim = None | |
|
580 | # arrayDim = None | |
|
581 | ||
|
582 | dsList = None #List of dictionaries | |
|
581 | 583 | |
|
582 | 584 | tableDim = None |
|
583 | 585 | |
@@ -611,9 +613,7 class HDF5Writer(Operation): | |||
|
611 | 613 | self.blocksPerFile = 10 |
|
612 | 614 | |
|
613 | 615 | self.metadataList = kwargs['metadataList'] |
|
614 | ||
|
615 | 616 | self.dataList = kwargs['dataList'] |
|
616 | ||
|
617 | 617 | self.dataOut = dataOut |
|
618 | 618 | |
|
619 | 619 | if kwargs.has_key('mode'): |
@@ -629,57 +629,91 class HDF5Writer(Operation): | |||
|
629 | 629 | arrayDim = numpy.zeros((len(self.dataList),5)) |
|
630 | 630 | |
|
631 | 631 | #Table dimensions |
|
632 | ||
|
633 | 632 | dtype0 = self.dtype |
|
634 | ||
|
635 | 633 | tableList = [] |
|
636 | 634 | |
|
635 | #Dictionary and list of tables | |
|
636 | dsList = [] | |
|
637 | ||
|
637 | 638 | for i in range(len(self.dataList)): |
|
638 | ||
|
639 | dsDict = {} | |
|
639 | 640 | dataAux = getattr(self.dataOut, self.dataList[i]) |
|
640 | ||
|
641 | dsDict['variable'] = self.dataList[i] | |
|
641 | 642 | #--------------------- Conditionals ------------------------ |
|
642 | 643 | #There is no data |
|
643 | 644 | if dataAux == None: |
|
644 | 645 | return 0 |
|
645 | 646 | |
|
646 | 647 | #Not array, just a number |
|
647 | if type(dataAux)==float or type(dataAux)==int: | |
|
648 | #Mode 0 | |
|
649 | if type(dataAux)==float or type(dataAux)==int: | |
|
650 | dsDict['mode'] = 0 | |
|
651 | dsDict['nDim'] = 0 | |
|
648 | 652 | arrayDim[i,0] = 1 |
|
649 | mode[i] = 0 | |
|
650 | ||
|
651 | #Mode meteors | |
|
652 |
elif mode[i] == 2: |
|
|
653 | dsList.append(dsDict) | |
|
654 | ||
|
655 | #Mode 2: meteors | |
|
656 | elif mode[i] == 2: | |
|
657 | # dsDict['nDim'] = 0 | |
|
658 | dsDict['dsName'] = 'table0' | |
|
659 | dsDict['mode'] = 2 # Mode meteors | |
|
660 | dsDict['shape'] = dataAux.shape[-1] | |
|
661 | dsDict['nDim'] = 0 | |
|
662 | ||
|
653 | 663 | arrayDim[i,3] = dataAux.shape[-1] |
|
654 | 664 | arrayDim[i,4] = mode[i] #Mode the data was stored |
|
665 | ||
|
666 | dsList.append(dsDict) | |
|
655 | 667 | |
|
656 |
# |
|
|
668 | #Mode 1 | |
|
657 | 669 | else: |
|
658 | 670 | arrayDim0 = dataAux.shape #Data dimensions |
|
659 | 671 | arrayDim[i,0] = len(arrayDim0) #Number of array dimensions |
|
660 | 672 | arrayDim[i,4] = mode[i] #Mode the data was stored |
|
661 | 673 | |
|
674 | strtable = 'table' | |
|
675 | dsDict['mode'] = 1 # Mode parameters | |
|
676 | ||
|
662 | 677 | # Three-dimension arrays |
|
663 | 678 | if len(arrayDim0) == 3: |
|
664 | 679 | arrayDim[i,1:-1] = numpy.array(arrayDim0) |
|
665 | ||
|
680 | nTables = int(arrayDim[i,2]) | |
|
681 | dsDict['dsNumber'] = nTables | |
|
682 | dsDict['shape'] = arrayDim[i,2:4] | |
|
683 | dsDict['nDim'] = 3 | |
|
684 | ||
|
685 | for j in range(nTables): | |
|
686 | dsDict = dsDict.copy() | |
|
687 | dsDict['dsName'] = strtable + str(j) | |
|
688 | dsList.append(dsDict) | |
|
689 | ||
|
666 | 690 | # Two-dimension arrays |
|
667 | 691 | elif len(arrayDim0) == 2: |
|
668 | 692 | arrayDim[i,2:-1] = numpy.array(arrayDim0) |
|
669 | ||
|
693 | nTables = int(arrayDim[i,2]) | |
|
694 | dsDict['dsNumber'] = nTables | |
|
695 | dsDict['shape'] = arrayDim[i,3] | |
|
696 | dsDict['nDim'] = 2 | |
|
697 | ||
|
698 | for j in range(nTables): | |
|
699 | dsDict = dsDict.copy() | |
|
700 | dsDict['dsName'] = strtable + str(j) | |
|
701 | dsList.append(dsDict) | |
|
702 | ||
|
670 | 703 | # One-dimension arrays |
|
671 | 704 | elif len(arrayDim0) == 1: |
|
672 | arrayDim[i,3] = arrayDim0 | |
|
673 | ||
|
674 | # No array, just a number | |
|
675 | elif len(arrayDim0) == 0: | |
|
676 |
|
|
|
677 | arrayDim[i,3] = 1 | |
|
705 | arrayDim[i,3] = arrayDim0[0] | |
|
706 | dsDict['shape'] = arrayDim0[0] | |
|
707 | dsDict['dsNumber'] = 1 | |
|
708 | dsDict['dsName'] = strtable + str(0) | |
|
709 | dsDict['nDim'] = 1 | |
|
710 | dsList.append(dsDict) | |
|
678 | 711 | |
|
679 | 712 | table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0) |
|
680 | 713 | tableList.append(table) |
|
681 | 714 | |
|
682 |
self.arrayDim = arrayDim |
|
|
715 | # self.arrayDim = arrayDim | |
|
716 | self.dsList = dsList | |
|
683 | 717 | self.tableDim = numpy.array(tableList, dtype = dtype0) |
|
684 | 718 | self.blockIndex = 0 |
|
685 | 719 | |
@@ -809,6 +843,117 class HDF5Writer(Operation): | |||
|
809 | 843 | |
|
810 | 844 | #Setting HDF5 File |
|
811 | 845 | fp = h5py.File(filename,'w') |
|
846 | #write metadata | |
|
847 | self.writeMetadata(fp) | |
|
848 | #Write data | |
|
849 | grp = fp.create_group("Data") | |
|
850 | # grp.attrs['metadata'] = self.metaFile | |
|
851 | ||
|
852 | # grp.attrs['blocksPerFile'] = 0 | |
|
853 | ds = [] | |
|
854 | data = [] | |
|
855 | dsList = self.dsList | |
|
856 | i = 0 | |
|
857 | while i < len(dsList): | |
|
858 | dsInfo = dsList[i] | |
|
859 | #One-dimension data | |
|
860 | if dsInfo['mode'] == 0: | |
|
861 | # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20') | |
|
862 | ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64) | |
|
863 | ds.append(ds0) | |
|
864 | data.append([]) | |
|
865 | i += 1 | |
|
866 | continue | |
|
867 | # nDimsForDs.append(nDims[i]) | |
|
868 | ||
|
869 | elif dsInfo['mode'] == 2: | |
|
870 | grp0 = grp.create_group(dsInfo['variable']) | |
|
871 | ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True) | |
|
872 | ds.append(ds0) | |
|
873 | data.append([]) | |
|
874 | i += 1 | |
|
875 | continue | |
|
876 | ||
|
877 | elif dsInfo['mode'] == 1: | |
|
878 | grp0 = grp.create_group(dsInfo['variable']) | |
|
879 | ||
|
880 | for j in range(dsInfo['dsNumber']): | |
|
881 | dsInfo = dsList[i] | |
|
882 | tableName = dsInfo['dsName'] | |
|
883 | shape = dsInfo['shape'] | |
|
884 | ||
|
885 | if dsInfo['nDim'] == 3: | |
|
886 | ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True) | |
|
887 | else: | |
|
888 | ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True) | |
|
889 | ||
|
890 | ds.append(ds0) | |
|
891 | data.append([]) | |
|
892 | i += 1 | |
|
893 | # nDimsForDs.append(nDims[i]) | |
|
894 | ||
|
895 | fp.flush() | |
|
896 | fp.close() | |
|
897 | ||
|
898 | # self.nDatas = nDatas | |
|
899 | # self.nDims = nDims | |
|
900 | # self.nDimsForDs = nDimsForDs | |
|
901 | #Saving variables | |
|
902 | print 'Writing the file: %s'%filename | |
|
903 | self.filename = filename | |
|
904 | # self.fp = fp | |
|
905 | # self.grp = grp | |
|
906 | # self.grp.attrs.modify('nRecords', 1) | |
|
907 | self.ds = ds | |
|
908 | self.data = data | |
|
909 | # self.setFile = setFile | |
|
910 | self.firsttime = True | |
|
911 | self.blockIndex = 0 | |
|
912 | return | |
|
913 | ||
|
914 | def setNextFile1(self): | |
|
915 | ||
|
916 | ext = self.ext | |
|
917 | path = self.path | |
|
918 | setFile = self.setFile | |
|
919 | mode = self.mode | |
|
920 | ||
|
921 | timeTuple = time.localtime(self.dataOut.utctime) | |
|
922 | subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) | |
|
923 | ||
|
924 | fullpath = os.path.join( path, subfolder ) | |
|
925 | ||
|
926 | if os.path.exists(fullpath): | |
|
927 | filesList = os.listdir( fullpath ) | |
|
928 | filesList = [k for k in filesList if 'D' in k] | |
|
929 | if len( filesList ) > 0: | |
|
930 | filesList = sorted( filesList, key=str.lower ) | |
|
931 | filen = filesList[-1] | |
|
932 | # el filename debera tener el siguiente formato | |
|
933 | # 0 1234 567 89A BCDE (hex) | |
|
934 | # x YYYY DDD SSS .ext | |
|
935 | if isNumber( filen[8:11] ): | |
|
936 | setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file | |
|
937 | else: | |
|
938 | setFile = -1 | |
|
939 | else: | |
|
940 | setFile = -1 #inicializo mi contador de seteo | |
|
941 | else: | |
|
942 | os.mkdir(fullpath) | |
|
943 | setFile = -1 #inicializo mi contador de seteo | |
|
944 | ||
|
945 | setFile += 1 | |
|
946 | ||
|
947 | file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, | |
|
948 | timeTuple.tm_year, | |
|
949 | timeTuple.tm_yday, | |
|
950 | setFile, | |
|
951 | ext ) | |
|
952 | ||
|
953 | filename = os.path.join( path, subfolder, file ) | |
|
954 | ||
|
955 | #Setting HDF5 File | |
|
956 | fp = h5py.File(filename,'w') | |
|
812 | 957 | |
|
813 | 958 | #writemetadata |
|
814 | 959 | self.writeMetadata(fp) |
@@ -914,6 +1059,7 class HDF5Writer(Operation): | |||
|
914 | 1059 | |
|
915 | 1060 | self.data |
|
916 | 1061 | ''' |
|
1062 | dsList = self.dsList | |
|
917 | 1063 | ds = self.ds |
|
918 | 1064 | #Setting HDF5 File |
|
919 | 1065 | fp = h5py.File(self.filename,'r+') |
@@ -921,25 +1067,20 class HDF5Writer(Operation): | |||
|
921 | 1067 | ind = 0 |
|
922 | 1068 | |
|
923 | 1069 | # grp.attrs['blocksPerFile'] = 0 |
|
924 |
|
|
|
1070 | while ind < len(dsList): | |
|
1071 | dsInfo = dsList[ind] | |
|
925 | 1072 | |
|
926 |
if |
|
|
927 |
ds0 = grp[ |
|
|
1073 | if dsInfo['mode'] == 0: | |
|
1074 | ds0 = grp[dsInfo['variable']] | |
|
928 | 1075 | ds[ind] = ds0 |
|
929 | 1076 | ind += 1 |
|
930 | 1077 | else: |
|
931 | # if self.mode[i] == 0: | |
|
932 | # strMode = "channel" | |
|
933 | if self.mode[i] == 1: | |
|
934 | strMode = "param" | |
|
935 | elif self.mode[i] == 2: | |
|
936 | strMode = "table" | |
|
937 | 1078 | |
|
938 |
grp0 = grp[ |
|
|
1079 | grp0 = grp[dsInfo['variable']] | |
|
939 | 1080 | |
|
940 |
for j in range( |
|
|
941 | tableName = strMode + str(j) | |
|
942 |
ds0 = grp0[ |
|
|
1081 | for j in range(dsInfo['dsNumber']): | |
|
1082 | dsInfo = dsList[ind] | |
|
1083 | ds0 = grp0[dsInfo['dsName']] | |
|
943 | 1084 | ds[ind] = ds0 |
|
944 | 1085 | ind += 1 |
|
945 | 1086 | |
@@ -957,33 +1098,32 class HDF5Writer(Operation): | |||
|
957 | 1098 | self.data |
|
958 | 1099 | ''' |
|
959 | 1100 | #Creating Arrays |
|
1101 | dsList = self.dsList | |
|
960 | 1102 | data = self.data |
|
961 | nDatas = self.nDatas | |
|
962 | nDims = self.nDims | |
|
963 | mode = self.mode | |
|
964 | 1103 | ind = 0 |
|
965 | 1104 | |
|
966 | for i in range(len(self.dataList)): | |
|
967 | dataAux = getattr(self.dataOut,self.dataList[i]) | |
|
1105 | while ind < len(dsList): | |
|
1106 | dsInfo = dsList[ind] | |
|
1107 | dataAux = getattr(self.dataOut, dsInfo['variable']) | |
|
968 | 1108 | |
|
969 | if nDims[i] == 1 or mode[i] == 2: | |
|
1109 | mode = dsInfo['mode'] | |
|
1110 | nDim = dsInfo['nDim'] | |
|
1111 | ||
|
1112 | if mode == 0 or mode == 2 or nDim == 1: | |
|
970 | 1113 | data[ind] = dataAux |
|
971 | ind += 1 | |
|
972 |
|
|
|
973 | elif nDims[i] == 2: | |
|
974 | for j in range(int(nDatas[i])): | |
|
1114 | ind += 1 | |
|
1115 | # elif nDim == 1: | |
|
1116 | # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1)) | |
|
1117 | # ind += 1 | |
|
1118 | elif nDim == 2: | |
|
1119 | for j in range(dsInfo['dsNumber']): | |
|
975 | 1120 | data[ind] = dataAux[j,:] |
|
976 | 1121 | ind += 1 |
|
977 |
|
|
|
978 | elif nDims[i] == 3: | |
|
979 | for j in range(int(nDatas[i])): | |
|
980 | # Extinct mode 0 | |
|
981 | # if (mode[i] == 0): | |
|
982 | # data[ind] = dataAux[j,:,:] | |
|
983 | # else: | |
|
1122 | elif nDim == 3: | |
|
1123 | for j in range(dsInfo['dsNumber']): | |
|
984 | 1124 | data[ind] = dataAux[:,j,:] |
|
985 | 1125 | ind += 1 |
|
986 | ||
|
1126 | ||
|
987 | 1127 | self.data = data |
|
988 | 1128 | return |
|
989 | 1129 | |
@@ -991,40 +1131,47 class HDF5Writer(Operation): | |||
|
991 | 1131 | ''' |
|
992 | 1132 | Saves the block in the HDF5 file |
|
993 | 1133 | ''' |
|
1134 | dsList = self.dsList | |
|
1135 | ||
|
994 | 1136 | for i in range(len(self.ds)): |
|
1137 | dsInfo = dsList[i] | |
|
1138 | nDim = dsInfo['nDim'] | |
|
1139 | mode = dsInfo['mode'] | |
|
995 | 1140 | |
|
996 | 1141 | # First time |
|
997 | 1142 | if self.firsttime: |
|
998 | 1143 | # self.ds[i].resize(self.data[i].shape) |
|
999 | 1144 | # self.ds[i][self.blockIndex,:] = self.data[i] |
|
1000 | 1145 | if type(self.data[i]) == numpy.ndarray: |
|
1001 | nDims1 = len(self.ds[i].shape) | |
|
1002 | 1146 | |
|
1003 |
if nDim |
|
|
1147 | if nDim == 3: | |
|
1004 | 1148 | self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1)) |
|
1005 | ||
|
1006 | self.ds[i].resize(self.data[i].shape) | |
|
1149 | self.ds[i].resize(self.data[i].shape) | |
|
1007 | 1150 | |
|
1008 | 1151 | self.ds[i][:] = self.data[i] |
|
1009 | 1152 | else: |
|
1010 | 1153 | |
|
1011 | 1154 | # From second time |
|
1012 | 1155 | # Meteors! |
|
1013 |
if |
|
|
1156 | if mode == 2: | |
|
1014 | 1157 | dataShape = self.data[i].shape |
|
1015 | 1158 | dsShape = self.ds[i].shape |
|
1016 | 1159 | self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1])) |
|
1017 | 1160 | self.ds[i][dsShape[0]:,:] = self.data[i] |
|
1018 |
# |
|
|
1019 |
elif |
|
|
1161 | # No dimension | |
|
1162 | elif mode == 0: | |
|
1020 | 1163 | self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1)) |
|
1021 | 1164 | self.ds[i][0,-1] = self.data[i] |
|
1165 | # One dimension | |
|
1166 | elif nDim == 1: | |
|
1167 | self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1])) | |
|
1168 | self.ds[i][-1,:] = self.data[i] | |
|
1022 | 1169 | # Two dimension |
|
1023 |
elif |
|
|
1170 | elif nDim == 2: | |
|
1024 | 1171 | self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1])) |
|
1025 | 1172 | self.ds[i][self.blockIndex,:] = self.data[i] |
|
1026 | 1173 | # Three dimensions |
|
1027 |
elif |
|
|
1174 | elif nDim == 3: | |
|
1028 | 1175 | self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1)) |
|
1029 | 1176 | self.ds[i][:,:,-1] = self.data[i] |
|
1030 | 1177 |
General Comments 0
You need to be logged in to leave comments.
Login now