| @@ -16,6 +16,7 import numpy | |||
|
|
16 | 16 | from schainpy.model.proc.jroproc_base import ProcessingUnit |
|
|
17 | 17 | from schainpy.model.data.jrodata import Parameters |
|
|
18 | 18 | from schainpy.model.io.jroIO_base import JRODataReader, isNumber |
|
|
19 | from schainpy.utils import log | |
|
|
19 | 20 | |
|
|
20 | 21 | FILE_HEADER_STRUCTURE = numpy.dtype([ |
|
|
21 | 22 | ('FMN', '<u4'), |
| @@ -109,11 +110,14 class BLTRParamReader(JRODataReader, ProcessingUnit): | |||
|
|
109 | 110 | timezone=0, |
|
|
110 | 111 | status_value=0, |
|
|
111 | 112 | **kwargs): |
|
|
112 | ||
|
|
113 | ||
|
|
113 | 114 | self.path = path |
|
|
115 | self.startDate = startDate | |
|
|
116 | self.endDate = endDate | |
|
|
114 | 117 | self.startTime = startTime |
|
|
115 |
self.endTime = endTime |
|
|
|
118 | self.endTime = endTime | |
|
|
116 | 119 | self.status_value = status_value |
|
|
120 | self.datatime = datetime.datetime(1900,1,1) | |
|
|
117 | 121 | |
|
|
118 | 122 | if self.path is None: |
|
|
119 | 123 | raise ValueError, "The path is not valid" |
| @@ -143,7 +147,7 class BLTRParamReader(JRODataReader, ProcessingUnit): | |||
|
|
143 | 147 | |
|
|
144 | 148 | ''' |
|
|
145 | 149 | |
|
|
146 | print 'Searching file in %s ' % (path) | |
|
|
150 | log.success('Searching files in {} '.format(path), 'BLTRParamReader') | |
|
|
147 | 151 | foldercounter = 0 |
|
|
148 | 152 | fileList0 = glob.glob1(path, "*%s" % ext) |
|
|
149 | 153 | fileList0.sort() |
| @@ -180,12 +184,11 class BLTRParamReader(JRODataReader, ProcessingUnit): | |||
|
|
180 | 184 | file_id = self.fileIndex |
|
|
181 | 185 | |
|
|
182 | 186 | if file_id == len(self.fileList): |
|
|
183 |
|
|
|
|
184 | print 'Total number of file(s) read : {}'.format(self.fileIndex + 1) | |
|
|
187 | log.success('No more files in the folder', 'BLTRParamReader') | |
|
|
185 | 188 | self.flagNoMoreFiles = 1 |
|
|
186 | 189 | return 0 |
|
|
187 | 190 | |
|
|
188 | print '\n[Setting file] (%s) ...' % self.fileList[file_id] | |
|
|
191 | log.success('Opening {}'.format(self.fileList[file_id]), 'BLTRParamReader') | |
|
|
189 | 192 | filename = os.path.join(self.path, self.fileList[file_id]) |
|
|
190 | 193 | |
|
|
191 | 194 | dirname, name = os.path.split(filename) |
| @@ -205,26 +208,29 class BLTRParamReader(JRODataReader, ProcessingUnit): | |||
|
|
205 | 208 | |
|
|
206 | 209 | def readNextBlock(self): |
|
|
207 | 210 | |
|
|
208 |
while True: |
|
|
|
211 | while True: | |
|
|
209 | 212 | if self.counter_records == self.nrecords: |
|
|
210 | 213 | self.flagIsNewFile = 1 |
|
|
211 | 214 | if not self.setNextFile(): |
|
|
212 | 215 | return 0 |
|
|
213 | ||
|
|
216 | ||
|
|
214 | 217 | self.readBlock() |
|
|
215 | ||
|
|
216 | if (self.datatime.time() < self.startTime) or (self.datatime.time() > self.endTime): | |
|
|
217 | print "[Reading] Record No. %d/%d -> %s [Skipping]" %( | |
|
|
218 | self.counter_records, | |
|
|
219 | self.nrecords, | |
|
|
220 | self.datatime.ctime()) | |
|
|
218 | ||
|
|
219 | if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \ | |
|
|
220 | (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)): | |
|
|
221 | log.warning( | |
|
|
222 | 'Reading Record No. {}/{} -> {} [Skipping]'.format( | |
|
|
223 | self.counter_records, | |
|
|
224 | self.nrecords, | |
|
|
225 | self.datatime.ctime()), | |
|
|
226 | 'BLTRParamReader') | |
|
|
221 | 227 | continue |
|
|
222 | 228 | break |
|
|
223 | 229 | |
|
|
224 |
|
|
|
|
230 | log.log('Reading Record No. {}/{} -> {}'.format( | |
|
|
225 | 231 | self.counter_records, |
|
|
226 | 232 | self.nrecords, |
|
|
227 | self.datatime.ctime()) | |
|
|
233 | self.datatime.ctime()), 'BLTRParamReader') | |
|
|
228 | 234 | |
|
|
229 | 235 | return 1 |
|
|
230 | 236 | |
| @@ -240,6 +246,7 class BLTRParamReader(JRODataReader, ProcessingUnit): | |||
|
|
240 | 246 | self.height = numpy.empty((self.nmodes, self.nranges)) |
|
|
241 | 247 | self.snr = numpy.empty((self.nmodes, self.nchannels, self.nranges)) |
|
|
242 | 248 | self.buffer = numpy.empty((self.nmodes, 3, self.nranges)) |
|
|
249 | self.flagDiscontinuousBlock = 0 | |
|
|
243 | 250 | |
|
|
244 | 251 | for mode in range(self.nmodes): |
|
|
245 | 252 | self.readHeader() |
| @@ -273,14 +280,11 class BLTRParamReader(JRODataReader, ProcessingUnit): | |||
|
|
273 | 280 | self.imode = self.header_rec['dmode_index'][0] |
|
|
274 | 281 | self.antenna = self.header_rec['antenna_coord'] |
|
|
275 | 282 | self.rx_gains = self.header_rec['rx_gains'] |
|
|
276 | self.time = self.header_rec['time'][0] | |
|
|
277 | tseconds = self.header_rec['time'][0] | |
|
|
278 | local_t1 = time.localtime(tseconds) | |
|
|
279 | self.year = local_t1.tm_year | |
|
|
280 | self.month = local_t1.tm_mon | |
|
|
281 | self.day = local_t1.tm_mday | |
|
|
282 | self.t = datetime.datetime(self.year, self.month, self.day) | |
|
|
283 | self.datatime = datetime.datetime.utcfromtimestamp(self.time) | |
|
|
283 | self.time = self.header_rec['time'][0] | |
|
|
284 | dt = datetime.datetime.utcfromtimestamp(self.time) | |
|
|
285 | if dt.date()>self.datatime.date(): | |
|
|
286 | self.flagDiscontinuousBlock = 1 | |
|
|
287 | self.datatime = dt | |
|
|
284 | 288 | |
|
|
285 | 289 | def readData(self): |
|
|
286 | 290 | ''' |
| @@ -322,7 +326,7 class BLTRParamReader(JRODataReader, ProcessingUnit): | |||
|
|
322 | 326 | |
|
|
323 | 327 | self.dataOut.data_SNR = self.snr |
|
|
324 | 328 | self.dataOut.height = self.height |
|
|
325 |
self.dataOut.data |
|
|
|
329 | self.dataOut.data = self.buffer | |
|
|
326 | 330 | self.dataOut.utctimeInit = self.time |
|
|
327 | 331 | self.dataOut.utctime = self.dataOut.utctimeInit |
|
|
328 | 332 | self.dataOut.useLocalTime = False |
| @@ -334,8 +338,7 class BLTRParamReader(JRODataReader, ProcessingUnit): | |||
|
|
334 | 338 | self.dataOut.lat = self.lat |
|
|
335 | 339 | self.dataOut.lon = self.lon |
|
|
336 | 340 | self.dataOut.channelList = range(self.nchannels) |
|
|
337 | self.dataOut.kchan = self.kchan | |
|
|
338 | # self.dataOut.nHeights = self.nranges | |
|
|
341 | self.dataOut.kchan = self.kchan | |
|
|
339 | 342 | self.dataOut.delta = self.delta |
|
|
340 | 343 | self.dataOut.correction = self.correction |
|
|
341 | 344 | self.dataOut.nmodes = self.nmodes |
| @@ -343,6 +346,7 class BLTRParamReader(JRODataReader, ProcessingUnit): | |||
|
|
343 | 346 | self.dataOut.antenna = self.antenna |
|
|
344 | 347 | self.dataOut.rx_gains = self.rx_gains |
|
|
345 | 348 | self.dataOut.flagNoData = False |
|
|
349 | self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock | |
|
|
346 | 350 | |
|
|
347 | 351 | def getData(self): |
|
|
348 | 352 | ''' |
| @@ -350,7 +354,7 class BLTRParamReader(JRODataReader, ProcessingUnit): | |||
|
|
350 | 354 | ''' |
|
|
351 | 355 | if self.flagNoMoreFiles: |
|
|
352 | 356 | self.dataOut.flagNoData = True |
|
|
353 |
|
|
|
|
357 | log.success('No file left to process', 'BLTRParamReader') | |
|
|
354 | 358 | return 0 |
|
|
355 | 359 | |
|
|
356 | 360 | if not self.readNextBlock(): |
| @@ -65,8 +65,8 class BLTRParametersProc(ProcessingUnit): | |||
|
|
65 | 65 | |
|
|
66 | 66 | if self.dataIn.type == 'Parameters': |
|
|
67 | 67 | self.dataOut.copy(self.dataIn) |
|
|
68 | ||
|
|
69 |
self.dataOut.data_ |
|
|
|
68 | ||
|
|
69 | self.dataOut.data_param = self.dataOut.data[mode] | |
|
|
70 | 70 | self.dataOut.heightList = self.dataOut.height[0] |
|
|
71 | 71 | self.dataOut.data_SNR = self.dataOut.data_SNR[mode] |
|
|
72 | 72 | |
| @@ -74,7 +74,7 class BLTRParametersProc(ProcessingUnit): | |||
|
|
74 | 74 | SNRavg = numpy.average(self.dataOut.data_SNR, axis=0) |
|
|
75 | 75 | SNRavgdB = 10*numpy.log10(SNRavg) |
|
|
76 | 76 | for i in range(3): |
|
|
77 |
self.dataOut.data_ |
|
|
|
77 | self.dataOut.data_param[i][SNRavgdB <= snr_threshold] = numpy.nan | |
|
|
78 | 78 | |
|
|
79 | 79 | # TODO |
|
|
80 | 80 | class OutliersFilter(Operation): |
General Comments 0
You need to be logged in to leave comments.
Login now