| @@ -691,6 +691,7 class Reader(object): | |||
|
|
691 | 691 | """Check if the given datetime is in range""" |
|
|
692 | 692 | startDateTime= datetime.datetime.combine(startDate,startTime) |
|
|
693 | 693 | endDateTime = datetime.datetime.combine(endDate,endTime) |
|
|
694 | #print("dt eval: ", dt, startDateTime,endDateTime) | |
|
|
694 | 695 | if startDateTime <= dt <= endDateTime: |
|
|
695 | 696 | return True |
|
|
696 | 697 | return False |
| @@ -154,7 +154,7 class HDFReader(Reader, ProcessingUnit): | |||
|
|
154 | 154 | self.dataOut = eval(self.meta['type'])() |
|
|
155 | 155 | |
|
|
156 | 156 | for attr in self.meta: |
|
|
157 | print("attr: ", attr) | |
|
|
157 | #print("attr: ", attr) | |
|
|
158 | 158 | setattr(self.dataOut, attr, self.meta[attr]) |
|
|
159 | 159 | |
|
|
160 | 160 | |
| @@ -273,7 +273,8 class HDFReader(Reader, ProcessingUnit): | |||
|
|
273 | 273 | def getData(self): |
|
|
274 | 274 | if not self.isDateTimeInRange(self.startFileDatetime, self.startDate, self.endDate, self.startTime, self.endTime): |
|
|
275 | 275 | self.dataOut.flagNoData = True |
|
|
276 | self.dataOut.error = True | |
|
|
276 | self.blockIndex = self.blocksPerFile | |
|
|
277 | #self.dataOut.error = True TERMINA EL PROGRAMA, removido | |
|
|
277 | 278 | return |
|
|
278 | 279 | for attr in self.data: |
|
|
279 | 280 | if self.data[attr].ndim == 1: |
| @@ -937,6 +937,332 class CleanRayleigh(Operation): | |||
|
|
937 | 937 | # plt.show() |
|
|
938 | 938 | return array |
|
|
939 | 939 | |
|
|
940 | ||
|
|
941 | class IntegrationFaradaySpectra(Operation): | |
|
|
942 | ||
|
|
943 | __profIndex = 0 | |
|
|
944 | __withOverapping = False | |
|
|
945 | ||
|
|
946 | __byTime = False | |
|
|
947 | __initime = None | |
|
|
948 | __lastdatatime = None | |
|
|
949 | __integrationtime = None | |
|
|
950 | ||
|
|
951 | __buffer_spc = None | |
|
|
952 | __buffer_cspc = None | |
|
|
953 | __buffer_dc = None | |
|
|
954 | ||
|
|
955 | __dataReady = False | |
|
|
956 | ||
|
|
957 | __timeInterval = None | |
|
|
958 | ||
|
|
959 | n = None | |
|
|
960 | ||
|
|
961 | def __init__(self): | |
|
|
962 | ||
|
|
963 | Operation.__init__(self) | |
|
|
964 | ||
|
|
965 | def setup(self, dataOut,n=None, timeInterval=None, overlapping=False, DPL=None): | |
|
|
966 | """ | |
|
|
967 | Set the parameters of the integration class. | |
|
|
968 | ||
|
|
969 | Inputs: | |
|
|
970 | ||
|
|
971 | n : Number of coherent integrations | |
|
|
972 | timeInterval : Time of integration. If the parameter "n" is selected this one does not work | |
|
|
973 | overlapping : | |
|
|
974 | ||
|
|
975 | """ | |
|
|
976 | ||
|
|
977 | self.__initime = None | |
|
|
978 | self.__lastdatatime = 0 | |
|
|
979 | ||
|
|
980 | self.__buffer_spc = [] | |
|
|
981 | self.__buffer_cspc = [] | |
|
|
982 | self.__buffer_dc = 0 | |
|
|
983 | ||
|
|
984 | self.__profIndex = 0 | |
|
|
985 | self.__dataReady = False | |
|
|
986 | self.__byTime = False | |
|
|
987 | ||
|
|
988 | #self.ByLags = dataOut.ByLags ###REDEFINIR | |
|
|
989 | self.ByLags = False | |
|
|
990 | ||
|
|
991 | if DPL != None: | |
|
|
992 | self.DPL=DPL | |
|
|
993 | else: | |
|
|
994 | #self.DPL=dataOut.DPL ###REDEFINIR | |
|
|
995 | self.DPL=0 | |
|
|
996 | ||
|
|
997 | if n is None and timeInterval is None: | |
|
|
998 | raise ValueError("n or timeInterval should be specified ...") | |
|
|
999 | ||
|
|
1000 | if n is not None: | |
|
|
1001 | self.n = int(n) | |
|
|
1002 | else: | |
|
|
1003 | ||
|
|
1004 | self.__integrationtime = int(timeInterval) | |
|
|
1005 | self.n = None | |
|
|
1006 | self.__byTime = True | |
|
|
1007 | ||
|
|
1008 | def putData(self, data_spc, data_cspc, data_dc): | |
|
|
1009 | """ | |
|
|
1010 | Add a profile to the __buffer_spc and increase in one the __profileIndex | |
|
|
1011 | ||
|
|
1012 | """ | |
|
|
1013 | ||
|
|
1014 | self.__buffer_spc.append(data_spc) | |
|
|
1015 | ||
|
|
1016 | if data_cspc is None: | |
|
|
1017 | self.__buffer_cspc = None | |
|
|
1018 | else: | |
|
|
1019 | self.__buffer_cspc.append(data_cspc) | |
|
|
1020 | ||
|
|
1021 | if data_dc is None: | |
|
|
1022 | self.__buffer_dc = None | |
|
|
1023 | else: | |
|
|
1024 | self.__buffer_dc += data_dc | |
|
|
1025 | ||
|
|
1026 | self.__profIndex += 1 | |
|
|
1027 | ||
|
|
1028 | return | |
|
|
1029 | ||
|
|
1030 | def hildebrand_sekhon_Integration(self,data,navg): | |
|
|
1031 | ||
|
|
1032 | sortdata = numpy.sort(data, axis=None) | |
|
|
1033 | sortID=data.argsort() | |
|
|
1034 | lenOfData = len(sortdata) | |
|
|
1035 | nums_min = lenOfData*0.75 | |
|
|
1036 | if nums_min <= 5: | |
|
|
1037 | nums_min = 5 | |
|
|
1038 | sump = 0. | |
|
|
1039 | sumq = 0. | |
|
|
1040 | j = 0 | |
|
|
1041 | cont = 1 | |
|
|
1042 | while((cont == 1)and(j < lenOfData)): | |
|
|
1043 | sump += sortdata[j] | |
|
|
1044 | sumq += sortdata[j]**2 | |
|
|
1045 | if j > nums_min: | |
|
|
1046 | rtest = float(j)/(j-1) + 1.0/navg | |
|
|
1047 | if ((sumq*j) > (rtest*sump**2)): | |
|
|
1048 | j = j - 1 | |
|
|
1049 | sump = sump - sortdata[j] | |
|
|
1050 | sumq = sumq - sortdata[j]**2 | |
|
|
1051 | cont = 0 | |
|
|
1052 | j += 1 | |
|
|
1053 | #lnoise = sump / j | |
|
|
1054 | ||
|
|
1055 | return j,sortID | |
|
|
1056 | ||
|
|
1057 | def pushData(self): | |
|
|
1058 | """ | |
|
|
1059 | Return the sum of the last profiles and the profiles used in the sum. | |
|
|
1060 | ||
|
|
1061 | Affected: | |
|
|
1062 | ||
|
|
1063 | self.__profileIndex | |
|
|
1064 | ||
|
|
1065 | """ | |
|
|
1066 | bufferH=None | |
|
|
1067 | buffer=None | |
|
|
1068 | buffer1=None | |
|
|
1069 | buffer_cspc=None | |
|
|
1070 | self.__buffer_spc=numpy.array(self.__buffer_spc) | |
|
|
1071 | self.__buffer_cspc=numpy.array(self.__buffer_cspc) | |
|
|
1072 | freq_dc = int(self.__buffer_spc.shape[2] / 2) | |
|
|
1073 | #print("FREQ_DC",freq_dc,self.__buffer_spc.shape,self.nHeights) | |
|
|
1074 | for k in range(7,self.nHeights): | |
|
|
1075 | buffer_cspc=numpy.copy(self.__buffer_cspc[:,:,:,k]) | |
|
|
1076 | outliers_IDs_cspc=[] | |
|
|
1077 | cspc_outliers_exist=False | |
|
|
1078 | #print("AQUIII") | |
|
|
1079 | for i in range(self.nChannels):#dataOut.nChannels): | |
|
|
1080 | ||
|
|
1081 | buffer1=numpy.copy(self.__buffer_spc[:,i,:,k]) | |
|
|
1082 | indexes=[] | |
|
|
1083 | #sortIDs=[] | |
|
|
1084 | outliers_IDs=[] | |
|
|
1085 | ||
|
|
1086 | for j in range(self.nProfiles): | |
|
|
1087 | # if i==0 and j==freq_dc: #NOT CONSIDERING DC PROFILE AT CHANNEL 0 | |
|
|
1088 | # continue | |
|
|
1089 | # if i==1 and j==0: #NOT CONSIDERING DC PROFILE AT CHANNEL 1 | |
|
|
1090 | # continue | |
|
|
1091 | buffer=buffer1[:,j] | |
|
|
1092 | index,sortID=self.hildebrand_sekhon_Integration(buffer,1) | |
|
|
1093 | ||
|
|
1094 | indexes.append(index) | |
|
|
1095 | #sortIDs.append(sortID) | |
|
|
1096 | outliers_IDs=numpy.append(outliers_IDs,sortID[index:]) | |
|
|
1097 | ||
|
|
1098 | outliers_IDs=numpy.array(outliers_IDs) | |
|
|
1099 | outliers_IDs=outliers_IDs.ravel() | |
|
|
1100 | outliers_IDs=numpy.unique(outliers_IDs) | |
|
|
1101 | outliers_IDs=outliers_IDs.astype(numpy.dtype('int64')) | |
|
|
1102 | indexes=numpy.array(indexes) | |
|
|
1103 | indexmin=numpy.min(indexes) | |
|
|
1104 | ||
|
|
1105 | if indexmin != buffer1.shape[0]: | |
|
|
1106 | cspc_outliers_exist=True | |
|
|
1107 | ###sortdata=numpy.sort(buffer1,axis=0) | |
|
|
1108 | ###avg2=numpy.mean(sortdata[:indexmin,:],axis=0) | |
|
|
1109 | lt=outliers_IDs | |
|
|
1110 | avg=numpy.mean(buffer1[[t for t in range(buffer1.shape[0]) if t not in lt],:],axis=0) | |
|
|
1111 | ||
|
|
1112 | for p in list(outliers_IDs): | |
|
|
1113 | buffer1[p,:]=avg | |
|
|
1114 | ||
|
|
1115 | self.__buffer_spc[:,i,:,k]=numpy.copy(buffer1) | |
|
|
1116 | ###cspc IDs | |
|
|
1117 | #indexmin_cspc+=indexmin_cspc | |
|
|
1118 | outliers_IDs_cspc=numpy.append(outliers_IDs_cspc,outliers_IDs) | |
|
|
1119 | ||
|
|
1120 | #if not breakFlag: | |
|
|
1121 | outliers_IDs_cspc=outliers_IDs_cspc.astype(numpy.dtype('int64')) | |
|
|
1122 | if cspc_outliers_exist: | |
|
|
1123 | #sortdata=numpy.sort(buffer_cspc,axis=0) | |
|
|
1124 | #avg=numpy.mean(sortdata[:indexmin_cpsc,:],axis=0) | |
|
|
1125 | lt=outliers_IDs_cspc | |
|
|
1126 | ||
|
|
1127 | avg=numpy.mean(buffer_cspc[[t for t in range(buffer_cspc.shape[0]) if t not in lt],:],axis=0) | |
|
|
1128 | for p in list(outliers_IDs_cspc): | |
|
|
1129 | buffer_cspc[p,:]=avg | |
|
|
1130 | ||
|
|
1131 | self.__buffer_cspc[:,:,:,k]=numpy.copy(buffer_cspc) | |
|
|
1132 | #else: | |
|
|
1133 | #break | |
|
|
1134 | ||
|
|
1135 | ||
|
|
1136 | ||
|
|
1137 | ||
|
|
1138 | buffer=None | |
|
|
1139 | bufferH=None | |
|
|
1140 | buffer1=None | |
|
|
1141 | buffer_cspc=None | |
|
|
1142 | ||
|
|
1143 | #print("cpsc",self.__buffer_cspc[:,0,0,0,0]) | |
|
|
1144 | #print(self.__profIndex) | |
|
|
1145 | #exit() | |
|
|
1146 | ||
|
|
1147 | buffer=None | |
|
|
1148 | #print(self.__buffer_spc[:,1,3,20,0]) | |
|
|
1149 | #print(self.__buffer_spc[:,1,5,37,0]) | |
|
|
1150 | data_spc = numpy.sum(self.__buffer_spc,axis=0) | |
|
|
1151 | data_cspc = numpy.sum(self.__buffer_cspc,axis=0) | |
|
|
1152 | ||
|
|
1153 | #print(numpy.shape(data_spc)) | |
|
|
1154 | #data_spc[1,4,20,0]=numpy.nan | |
|
|
1155 | ||
|
|
1156 | #data_cspc = self.__buffer_cspc | |
|
|
1157 | data_dc = self.__buffer_dc | |
|
|
1158 | n = self.__profIndex | |
|
|
1159 | ||
|
|
1160 | self.__buffer_spc = [] | |
|
|
1161 | self.__buffer_cspc = [] | |
|
|
1162 | self.__buffer_dc = 0 | |
|
|
1163 | self.__profIndex = 0 | |
|
|
1164 | ||
|
|
1165 | return data_spc, data_cspc, data_dc, n | |
|
|
1166 | ||
|
|
1167 | def byProfiles(self, *args): | |
|
|
1168 | ||
|
|
1169 | self.__dataReady = False | |
|
|
1170 | avgdata_spc = None | |
|
|
1171 | avgdata_cspc = None | |
|
|
1172 | avgdata_dc = None | |
|
|
1173 | ||
|
|
1174 | self.putData(*args) | |
|
|
1175 | ||
|
|
1176 | if self.__profIndex == self.n: | |
|
|
1177 | ||
|
|
1178 | avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData() | |
|
|
1179 | self.n = n | |
|
|
1180 | self.__dataReady = True | |
|
|
1181 | ||
|
|
1182 | return avgdata_spc, avgdata_cspc, avgdata_dc | |
|
|
1183 | ||
|
|
1184 | def byTime(self, datatime, *args): | |
|
|
1185 | ||
|
|
1186 | self.__dataReady = False | |
|
|
1187 | avgdata_spc = None | |
|
|
1188 | avgdata_cspc = None | |
|
|
1189 | avgdata_dc = None | |
|
|
1190 | ||
|
|
1191 | self.putData(*args) | |
|
|
1192 | ||
|
|
1193 | if (datatime - self.__initime) >= self.__integrationtime: | |
|
|
1194 | avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData() | |
|
|
1195 | self.n = n | |
|
|
1196 | self.__dataReady = True | |
|
|
1197 | ||
|
|
1198 | return avgdata_spc, avgdata_cspc, avgdata_dc | |
|
|
1199 | ||
|
|
1200 | def integrate(self, datatime, *args): | |
|
|
1201 | ||
|
|
1202 | if self.__profIndex == 0: | |
|
|
1203 | self.__initime = datatime | |
|
|
1204 | ||
|
|
1205 | if self.__byTime: | |
|
|
1206 | avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime( | |
|
|
1207 | datatime, *args) | |
|
|
1208 | else: | |
|
|
1209 | avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args) | |
|
|
1210 | ||
|
|
1211 | if not self.__dataReady: | |
|
|
1212 | return None, None, None, None | |
|
|
1213 | ||
|
|
1214 | return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc | |
|
|
1215 | ||
|
|
1216 | def run(self, dataOut, n=None, DPL = None,timeInterval=None, overlapping=False): | |
|
|
1217 | if n == 1: | |
|
|
1218 | return dataOut | |
|
|
1219 | ||
|
|
1220 | dataOut.flagNoData = True | |
|
|
1221 | ||
|
|
1222 | if not self.isConfig: | |
|
|
1223 | self.setup(dataOut, n, timeInterval, overlapping,DPL ) | |
|
|
1224 | self.isConfig = True | |
|
|
1225 | ||
|
|
1226 | if not self.ByLags: | |
|
|
1227 | self.nProfiles=dataOut.nProfiles | |
|
|
1228 | self.nChannels=dataOut.nChannels | |
|
|
1229 | self.nHeights=dataOut.nHeights | |
|
|
1230 | avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime, | |
|
|
1231 | dataOut.data_spc, | |
|
|
1232 | dataOut.data_cspc, | |
|
|
1233 | dataOut.data_dc) | |
|
|
1234 | else: | |
|
|
1235 | self.nProfiles=dataOut.nProfiles | |
|
|
1236 | self.nChannels=dataOut.nChannels | |
|
|
1237 | self.nHeights=dataOut.nHeights | |
|
|
1238 | avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime, | |
|
|
1239 | dataOut.dataLag_spc, | |
|
|
1240 | dataOut.dataLag_cspc, | |
|
|
1241 | dataOut.dataLag_dc) | |
|
|
1242 | ||
|
|
1243 | if self.__dataReady: | |
|
|
1244 | ||
|
|
1245 | if not self.ByLags: | |
|
|
1246 | ||
|
|
1247 | dataOut.data_spc = numpy.squeeze(avgdata_spc) | |
|
|
1248 | dataOut.data_cspc = numpy.squeeze(avgdata_cspc) | |
|
|
1249 | dataOut.data_dc = avgdata_dc | |
|
|
1250 | else: | |
|
|
1251 | dataOut.dataLag_spc = avgdata_spc | |
|
|
1252 | dataOut.dataLag_cspc = avgdata_cspc | |
|
|
1253 | dataOut.dataLag_dc = avgdata_dc | |
|
|
1254 | ||
|
|
1255 | dataOut.data_spc=dataOut.dataLag_spc[:,:,:,dataOut.LagPlot] | |
|
|
1256 | dataOut.data_cspc=dataOut.dataLag_cspc[:,:,:,dataOut.LagPlot] | |
|
|
1257 | dataOut.data_dc=dataOut.dataLag_dc[:,:,dataOut.LagPlot] | |
|
|
1258 | ||
|
|
1259 | ||
|
|
1260 | dataOut.nIncohInt *= self.n | |
|
|
1261 | dataOut.utctime = avgdatatime | |
|
|
1262 | dataOut.flagNoData = False | |
|
|
1263 | ||
|
|
1264 | return dataOut | |
|
|
1265 | ||
|
|
940 | 1266 | class removeInterference(Operation): |
|
|
941 | 1267 | |
|
|
942 | 1268 | def removeInterference2(self): |
General Comments 0
You need to be logged in to leave comments.
Login now