##// END OF EJS Templates
schain
RSS
Bug fixes to HDF5 Writer Unit
Julio Valdez -
r820:e3a794954261
parent child
Show More
Show file before | Show file after | Hide comments
@@ -579,22 +579,14 class HDF5Writer(Operation):
579
579
580 # arrayDim = None
580 # arrayDim = None
581
581
582 dsList = None #List of dictionaries
582 dsList = None #List of dictionaries with dataset properties
583
583
584 tableDim = None
584 tableDim = None
585
585
586 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
586 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
587
587
588 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
588 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
589
589
590 mode = None
591
592 nDatas = None #Number of datasets to be stored per array
593
594 nDims = None #Number Dimensions in each dataset
595
596 nDimsForDs = None
597
598 currentDay = None
590 currentDay = None
599
591
600 def __init__(self):
592 def __init__(self):
@@ -649,7 +641,7 class HDF5Writer(Operation):
649 if type(dataAux)==float or type(dataAux)==int:
641 if type(dataAux)==float or type(dataAux)==int:
650 dsDict['mode'] = 0
642 dsDict['mode'] = 0
651 dsDict['nDim'] = 0
643 dsDict['nDim'] = 0
652 arrayDim[i,0] = 1
644 arrayDim[i,0] = 0
653 dsList.append(dsDict)
645 dsList.append(dsDict)
654
646
655 #Mode 2: meteors
647 #Mode 2: meteors
@@ -659,6 +651,7 class HDF5Writer(Operation):
659 dsDict['mode'] = 2 # Mode meteors
651 dsDict['mode'] = 2 # Mode meteors
660 dsDict['shape'] = dataAux.shape[-1]
652 dsDict['shape'] = dataAux.shape[-1]
661 dsDict['nDim'] = 0
653 dsDict['nDim'] = 0
654 dsDict['dsNumber'] = 1
662
655
663 arrayDim[i,3] = dataAux.shape[-1]
656 arrayDim[i,3] = dataAux.shape[-1]
664 arrayDim[i,4] = mode[i] #Mode the data was stored
657 arrayDim[i,4] = mode[i] #Mode the data was stored
@@ -911,134 +904,6 class HDF5Writer(Operation):
911 self.blockIndex = 0
904 self.blockIndex = 0
912 return
905 return
913
906
914 def setNextFile1(self):
915
916 ext = self.ext
917 path = self.path
918 setFile = self.setFile
919 mode = self.mode
920
921 timeTuple = time.localtime(self.dataOut.utctime)
922 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
923
924 fullpath = os.path.join( path, subfolder )
925
926 if os.path.exists(fullpath):
927 filesList = os.listdir( fullpath )
928 filesList = [k for k in filesList if 'D' in k]
929 if len( filesList ) > 0:
930 filesList = sorted( filesList, key=str.lower )
931 filen = filesList[-1]
932 # el filename debera tener el siguiente formato
933 # 0 1234 567 89A BCDE (hex)
934 # x YYYY DDD SSS .ext
935 if isNumber( filen[8:11] ):
936 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
937 else:
938 setFile = -1
939 else:
940 setFile = -1 #inicializo mi contador de seteo
941 else:
942 os.mkdir(fullpath)
943 setFile = -1 #inicializo mi contador de seteo
944
945 setFile += 1
946
947 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
948 timeTuple.tm_year,
949 timeTuple.tm_yday,
950 setFile,
951 ext )
952
953 filename = os.path.join( path, subfolder, file )
954
955 #Setting HDF5 File
956 fp = h5py.File(filename,'w')
957
958 #writemetadata
959 self.writeMetadata(fp)
960
961 grp = fp.create_group("Data")
962 # grp.attrs['metadata'] = self.metaFile
963
964 # grp.attrs['blocksPerFile'] = 0
965
966 ds = []
967 data = []
968 nDimsForDs = []
969
970 nDatas = numpy.zeros(len(self.dataList))
971 nDims = self.arrayDim[:,0]
972
973 nDim1 = self.arrayDim[:,2]
974 nDim0 = self.arrayDim[:,3]
975
976 for i in range(len(self.dataList)):
977
978 #One-dimension data
979 if nDims[i]==1:
980 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
981 ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
982 ds.append(ds0)
983 data.append([])
984 nDimsForDs.append(nDims[i])
985 else:
986
987 #Channel mode
988 # if mode[i] == 0:
989 # strMode = "channel"
990 #
991 # #nDatas is the number of arrays per variable
992 # if nDims[i] == 1:
993 # nDatas[i] = self.arrayDim[i,1]
994 # elif nDims[i] == 2:
995 # nDatas[i] = self.arrayDim[i,2]
996
997 #Parameters mode
998 if mode[i] == 1:
999 strMode = "param"
1000 nDatas[i] = self.arrayDim[i,2]
1001
1002 #Meteors mode
1003 elif mode[i] == 2:
1004 strMode = "table"
1005 nDatas[i] = 1
1006
1007 grp0 = grp.create_group(self.dataList[i])
1008
1009 for j in range(int(nDatas[i])):
1010 tableName = strMode + str(j)
1011
1012 if nDims[i] == 3:
1013 ds0 = grp0.create_dataset(tableName, (nDim1[i],nDim0[i],1) , data = numpy.zeros((nDim1[i],nDim0[i],1)) ,maxshape=(None,nDim0[i],None), chunks=True)
1014
1015 else:
1016 ds0 = grp0.create_dataset(tableName, (1,nDim0[i]), data = numpy.zeros((1,nDim0[i])) , maxshape=(None,nDim0[i]), chunks=True)
1017
1018 ds.append(ds0)
1019 data.append([])
1020 nDimsForDs.append(nDims[i])
1021
1022 fp.flush()
1023 fp.close()
1024
1025 self.nDatas = nDatas
1026 self.nDims = nDims
1027 self.nDimsForDs = nDimsForDs
1028 #Saving variables
1029 print 'Writing the file: %s'%filename
1030 self.filename = filename
1031 # self.fp = fp
1032 # self.grp = grp
1033 # self.grp.attrs.modify('nRecords', 1)
1034 self.ds = ds
1035 self.data = data
1036 #
1037 # self.setFile = setFile
1038 self.firsttime = True
1039 self.blockIndex = 0
1040 return
1041
1042 def putData(self):
907 def putData(self):
1043
908
1044 if self.blockIndex == self.blocksPerFile or self.dateFlag():
909 if self.blockIndex == self.blocksPerFile or self.dateFlag():
@@ -1147,7 +1012,8 class HDF5Writer(Operation):
1147 if nDim == 3:
1012 if nDim == 3:
1148 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1013 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1149 self.ds[i].resize(self.data[i].shape)
1014 self.ds[i].resize(self.data[i].shape)
1150
1015 if mode == 2:
1016 self.ds[i].resize(self.data[i].shape)
1151 self.ds[i][:] = self.data[i]
1017 self.ds[i][:] = self.data[i]
1152 else:
1018 else:
1153
1019
General Comments 0
You need to be logged in to leave comments. Login now