##// END OF EJS Templates
updates real multiprocess, save hdf5, utils_Io
joabAM -
r1559:6d931be610e9
parent child
Show More

The requested changes are too big and content was truncated. Show full diff

@@ -1,1136 +1,1154
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """Definition of diferent Data objects for different types of data
5 """Definition of diferent Data objects for different types of data
6
6
7 Here you will find the diferent data objects for the different types
7 Here you will find the diferent data objects for the different types
8 of data, this data objects must be used as dataIn or dataOut objects in
8 of data, this data objects must be used as dataIn or dataOut objects in
9 processing units and operations. Currently the supported data objects are:
9 processing units and operations. Currently the supported data objects are:
10 Voltage, Spectra, SpectraHeis, Fits, Correlation and Parameters
10 Voltage, Spectra, SpectraHeis, Fits, Correlation and Parameters
11 """
11 """
12
12
13 import copy
13 import copy
14 import numpy
14 import numpy
15 import datetime
15 import datetime
16 import json
16 import json
17
17
18 import schainpy.admin
18 import schainpy.admin
19 from schainpy.utils import log
19 from schainpy.utils import log
20 from .jroheaderIO import SystemHeader, RadarControllerHeader
20 from .jroheaderIO import SystemHeader, RadarControllerHeader,ProcessingHeader
21 from schainpy.model.data import _noise
21 from schainpy.model.data import _noise
22
22 SPEED_OF_LIGHT = 3e8
23
23
24 def getNumpyDtype(dataTypeCode):
24 def getNumpyDtype(dataTypeCode):
25
25
26 if dataTypeCode == 0:
26 if dataTypeCode == 0:
27 numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
27 numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
28 elif dataTypeCode == 1:
28 elif dataTypeCode == 1:
29 numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
29 numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
30 elif dataTypeCode == 2:
30 elif dataTypeCode == 2:
31 numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
31 numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
32 elif dataTypeCode == 3:
32 elif dataTypeCode == 3:
33 numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
33 numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
34 elif dataTypeCode == 4:
34 elif dataTypeCode == 4:
35 numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
35 numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
36 elif dataTypeCode == 5:
36 elif dataTypeCode == 5:
37 numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
37 numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
38 else:
38 else:
39 raise ValueError('dataTypeCode was not defined')
39 raise ValueError('dataTypeCode was not defined')
40
40
41 return numpyDtype
41 return numpyDtype
42
42
43
43
44 def getDataTypeCode(numpyDtype):
44 def getDataTypeCode(numpyDtype):
45
45
46 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
46 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
47 datatype = 0
47 datatype = 0
48 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
48 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
49 datatype = 1
49 datatype = 1
50 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
50 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
51 datatype = 2
51 datatype = 2
52 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
52 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
53 datatype = 3
53 datatype = 3
54 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
54 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
55 datatype = 4
55 datatype = 4
56 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
56 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
57 datatype = 5
57 datatype = 5
58 else:
58 else:
59 datatype = None
59 datatype = None
60
60
61 return datatype
61 return datatype
62
62
63
63
64 def hildebrand_sekhon(data, navg):
64 def hildebrand_sekhon(data, navg):
65 """
65 """
66 This method is for the objective determination of the noise level in Doppler spectra. This
66 This method is for the objective determination of the noise level in Doppler spectra. This
67 implementation technique is based on the fact that the standard deviation of the spectral
67 implementation technique is based on the fact that the standard deviation of the spectral
68 densities is equal to the mean spectral density for white Gaussian noise
68 densities is equal to the mean spectral density for white Gaussian noise
69
69
70 Inputs:
70 Inputs:
71 Data : heights
71 Data : heights
72 navg : numbers of averages
72 navg : numbers of averages
73
73
74 Return:
74 Return:
75 mean : noise's level
75 mean : noise's level
76 """
76 """
77
77
78 sortdata = numpy.sort(data, axis=None)
78 sortdata = numpy.sort(data, axis=None)
79 '''
79 '''
80 lenOfData = len(sortdata)
80 lenOfData = len(sortdata)
81 nums_min = lenOfData*0.5
81 nums_min = lenOfData*0.5
82
82
83 if nums_min <= 5:
83 if nums_min <= 5:
84
84
85 nums_min = 5
85 nums_min = 5
86
86
87 sump = 0.
87 sump = 0.
88 sumq = 0.
88 sumq = 0.
89
89
90 j = 0
90 j = 0
91 cont = 1
91 cont = 1
92
92
93 while((cont == 1)and(j < lenOfData)):
93 while((cont == 1)and(j < lenOfData)):
94
94
95 sump += sortdata[j]
95 sump += sortdata[j]
96 sumq += sortdata[j]**2
96 sumq += sortdata[j]**2
97
97
98 if j > nums_min:
98 if j > nums_min:
99 rtest = float(j)/(j-1) + 1.0/navg
99 rtest = float(j)/(j-1) + 1.0/navg
100 if ((sumq*j) > (rtest*sump**2)):
100 if ((sumq*j) > (rtest*sump**2)):
101 j = j - 1
101 j = j - 1
102 sump = sump - sortdata[j]
102 sump = sump - sortdata[j]
103 sumq = sumq - sortdata[j]**2
103 sumq = sumq - sortdata[j]**2
104 cont = 0
104 cont = 0
105
105
106 j += 1
106 j += 1
107
107
108 lnoise = sump / j
108 lnoise = sump / j
109 return lnoise
109 return lnoise
110 '''
110 '''
111 return _noise.hildebrand_sekhon(sortdata, navg)
111 return _noise.hildebrand_sekhon(sortdata, navg)
112
112
113
113
114 class Beam:
114 class Beam:
115
115
116 def __init__(self):
116 def __init__(self):
117 self.codeList = []
117 self.codeList = []
118 self.azimuthList = []
118 self.azimuthList = []
119 self.zenithList = []
119 self.zenithList = []
120
120
121
121
122
122
123 class GenericData(object):
123 class GenericData(object):
124
124
125 flagNoData = True
125 flagNoData = True
126
126
127 def copy(self, inputObj=None):
127 def copy(self, inputObj=None):
128
128
129 if inputObj == None:
129 if inputObj == None:
130 return copy.deepcopy(self)
130 return copy.deepcopy(self)
131
131
132 for key in list(inputObj.__dict__.keys()):
132 for key in list(inputObj.__dict__.keys()):
133
133
134 attribute = inputObj.__dict__[key]
134 attribute = inputObj.__dict__[key]
135
135
136 # If this attribute is a tuple or list
136 # If this attribute is a tuple or list
137 if type(inputObj.__dict__[key]) in (tuple, list):
137 if type(inputObj.__dict__[key]) in (tuple, list):
138 self.__dict__[key] = attribute[:]
138 self.__dict__[key] = attribute[:]
139 continue
139 continue
140
140
141 # If this attribute is another object or instance
141 # If this attribute is another object or instance
142 if hasattr(attribute, '__dict__'):
142 if hasattr(attribute, '__dict__'):
143 self.__dict__[key] = attribute.copy()
143 self.__dict__[key] = attribute.copy()
144 continue
144 continue
145
145
146 self.__dict__[key] = inputObj.__dict__[key]
146 self.__dict__[key] = inputObj.__dict__[key]
147
147
148 def deepcopy(self):
148 def deepcopy(self):
149
149
150 return copy.deepcopy(self)
150 return copy.deepcopy(self)
151
151
152 def isEmpty(self):
152 def isEmpty(self):
153
153
154 return self.flagNoData
154 return self.flagNoData
155
155
156 def isReady(self):
156 def isReady(self):
157
157
158 return not self.flagNoData
158 return not self.flagNoData
159
159
160
160
161 class JROData(GenericData):
161 class JROData(GenericData):
162
162
163 useInputBuffer = False
163 useInputBuffer = False
164 buffer_empty = True
164 buffer_empty = True
165
165
166 systemHeaderObj = SystemHeader()
166 systemHeaderObj = SystemHeader()
167 radarControllerHeaderObj = RadarControllerHeader()
167 radarControllerHeaderObj = RadarControllerHeader()
168 type = None
168 type = None
169 datatype = None # dtype but in string
169 datatype = None # dtype but in string
170 nProfiles = None
170 nProfiles = None
171 heightList = None
171 heightList = None
172 channelList = None
172 channelList = None
173 flagDiscontinuousBlock = False
173 flagDiscontinuousBlock = False
174 useLocalTime = False
174 useLocalTime = False
175 utctime = None
175 utctime = None
176 timeZone = None
176 timeZone = None
177 dstFlag = None
177 dstFlag = None
178 errorCount = None
178 errorCount = None
179 blocksize = None
179 blocksize = None
180 flagDecodeData = False # asumo q la data no esta decodificada
180 flagDecodeData = False # asumo q la data no esta decodificada
181 flagDeflipData = False # asumo q la data no esta sin flip
181 flagDeflipData = False # asumo q la data no esta sin flip
182 flagShiftFFT = False
182 flagShiftFFT = False
183 nCohInt = None
183 nCohInt = None
184 windowOfFilter = 1
184 windowOfFilter = 1
185 C = 3e8
185 C = 3e8
186 frequency = 49.92e6
186 frequency = 49.92e6
187 realtime = False
187 realtime = False
188 beacon_heiIndexList = None
188 beacon_heiIndexList = None
189 last_block = None
189 last_block = None
190 blocknow = None
190 blocknow = None
191 azimuth = None
191 azimuth = None
192 zenith = None
192 zenith = None
193 beam = Beam()
193
194 profileIndex = None
194 profileIndex = None
195 error = None
195 error = None
196 data = None
196 data = None
197 nmodes = None
197 nmodes = None
198 metadata_list = ['heightList', 'timeZone', 'type']
198 metadata_list = ['heightList', 'timeZone', 'type']
199 codeList = []
199 codeList = []
200 azimuthList = []
200 azimuthList = []
201 elevationList = []
201 elevationList = []
202 last_noise = None
202 last_noise = None
203 radar_ipp = None
203 __ipp = None
204 __ippSeconds = None
204 sampled_heightsFFT = None
205 sampled_heightsFFT = None
205 pulseLength_TxA = None
206 pulseLength_TxA = None
206 deltaHeight = None
207 deltaHeight = None
208 code = None
209 nCode = None
210 nBaud = None
211 unitsDescription = "The units of the parameters are according to the International System of units (Seconds, Meter, Hertz, ...), except \
212 the parameters related to distances such as heightList, or heightResolution wich are in Km"
213
207
214
208 def __str__(self):
215 def __str__(self):
209
216
210 return '{} - {}'.format(self.type, self.datatime())
217 return '{} - {}'.format(self.type, self.datatime())
211
218
212 def getNoise(self):
219 def getNoise(self):
213
220
214 raise NotImplementedError
221 raise NotImplementedError
215
222
216 @property
223 @property
217 def nChannels(self):
224 def nChannels(self):
218
225
219 return len(self.channelList)
226 return len(self.channelList)
220
227
221 @property
228 @property
222 def channelIndexList(self):
229 def channelIndexList(self):
223
230
224 return list(range(self.nChannels))
231 return list(range(self.nChannels))
225
232
226 @property
233 @property
227 def nHeights(self):
234 def nHeights(self):
228
235
229 return len(self.heightList)
236 return len(self.heightList)
230
237
231 def getDeltaH(self):
238 def getDeltaH(self):
232
239
233 return self.heightList[1] - self.heightList[0]
240 return self.heightList[1] - self.heightList[0]
234
241
235 @property
242 @property
236 def ltctime(self):
243 def ltctime(self):
237
244
238 if self.useLocalTime:
245 if self.useLocalTime:
239 return self.utctime - self.timeZone * 60
246 return self.utctime - self.timeZone * 60
240
247
241 return self.utctime
248 return self.utctime
242
249
243 @property
250 @property
244 def datatime(self):
251 def datatime(self):
245
252
246 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
253 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
247 return datatimeValue
254 return datatimeValue
248
255
249 def getTimeRange(self):
256 def getTimeRange(self):
250
257
251 datatime = []
258 datatime = []
252
259
253 datatime.append(self.ltctime)
260 datatime.append(self.ltctime)
254 datatime.append(self.ltctime + self.timeInterval + 1)
261 datatime.append(self.ltctime + self.timeInterval + 1)
255
262
256 datatime = numpy.array(datatime)
263 datatime = numpy.array(datatime)
257
264
258 return datatime
265 return datatime
259
266
260 def getFmaxTimeResponse(self):
267 def getFmaxTimeResponse(self):
261
268
262 period = (10**-6) * self.getDeltaH() / (0.15)
269 period = (10**-6) * self.getDeltaH() / (0.15)
263
270
264 PRF = 1. / (period * self.nCohInt)
271 PRF = 1. / (period * self.nCohInt)
265
272
266 fmax = PRF
273 fmax = PRF
267
274
268 return fmax
275 return fmax
269
276
270 def getFmax(self):
277 def getFmax(self):
271 PRF = 1. / (self.ippSeconds * self.nCohInt)
278 PRF = 1. / (self.__ippSeconds * self.nCohInt)
272
279
273 fmax = PRF
280 fmax = PRF
274 return fmax
281 return fmax
275
282
276 def getVmax(self):
283 def getVmax(self):
277
284
278 _lambda = self.C / self.frequency
285 _lambda = self.C / self.frequency
279
286
280 vmax = self.getFmax() * _lambda / 2
287 vmax = self.getFmax() * _lambda / 2
281
288
282 return vmax
289 return vmax
283
290
284 @property
291 @property
285 def ippSeconds(self):
292 def ippSeconds(self):
286 '''
293 '''
287 '''
294 '''
288 return self.radarControllerHeaderObj.ippSeconds
295 #return self.radarControllerHeaderObj.ippSeconds
296 return self.__ippSeconds
289
297
290 @ippSeconds.setter
298 @ippSeconds.setter
291 def ippSeconds(self, ippSeconds):
299 def ippSeconds(self, ippSeconds):
292 '''
300 '''
293 '''
301 '''
294 self.radarControllerHeaderObj.ippSeconds = ippSeconds
302 #self.radarControllerHeaderObj.ippSeconds = ippSeconds
295
303 self.__ippSeconds = ippSeconds
296 @property
304 self.__ipp = ippSeconds*SPEED_OF_LIGHT/2000.0
297 def code(self):
298 '''
299 '''
300 return self.radarControllerHeaderObj.code
301
302 @code.setter
303 def code(self, code):
304 '''
305 '''
306 self.radarControllerHeaderObj.code = code
307
305
308 @property
309 def nCode(self):
310 '''
311 '''
312 return self.radarControllerHeaderObj.nCode
313
314 @nCode.setter
315 def nCode(self, ncode):
316 '''
317 '''
318 self.radarControllerHeaderObj.nCode = ncode
319
320 @property
321 def nBaud(self):
322 '''
323 '''
324 return self.radarControllerHeaderObj.nBaud
325
306
326 @nBaud.setter
307 # @property
327 def nBaud(self, nbaud):
308 # def code(self):
328 '''
309 # '''
329 '''
310 # '''
330 self.radarControllerHeaderObj.nBaud = nbaud
311 # return self.radarControllerHeaderObj.code
312 #
313 # @code.setter
314 # def code(self, code):
315 # '''
316 # '''
317 # self.radarControllerHeaderObj.code = code
318 #
319 # @property
320 # def nCode(self):
321 # '''
322 # '''
323 # return self.radarControllerHeaderObj.nCode
324 #
325 # @nCode.setter
326 # def nCode(self, ncode):
327 # '''
328 # '''
329 # self.radarControllerHeaderObj.nCode = ncode
330 #
331 # @property
332 # def nBaud(self):
333 # '''
334 # '''
335 # return self.radarControllerHeaderObj.nBaud
336 #
337 # @nBaud.setter
338 # def nBaud(self, nbaud):
339 # '''
340 # '''
341 # self.radarControllerHeaderObj.nBaud = nbaud
331
342
332 @property
343 @property
333 def ipp(self):
344 def ipp(self):
334 '''
345 '''
335 '''
346 '''
336 return self.radarControllerHeaderObj.ipp
347 return self.__ipp
348 #return self.radarControllerHeaderObj.ipp
337
349
338 @ipp.setter
350 @ipp.setter
339 def ipp(self, ipp):
351 def ipp(self, ipp):
340 '''
352 '''
341 '''
353 '''
342 self.radarControllerHeaderObj.ipp = ipp
354 self.__ipp = ipp
355 #self.radarControllerHeaderObj.ipp = ipp
343
356
344 @property
357 @property
345 def metadata(self):
358 def metadata(self):
346 '''
359 '''
347 '''
360 '''
348
361
349 return {attr: getattr(self, attr) for attr in self.metadata_list}
362 return {attr: getattr(self, attr) for attr in self.metadata_list}
350
363
351
364
352 class Voltage(JROData):
365 class Voltage(JROData):
353
366
354 dataPP_POW = None
367 dataPP_POW = None
355 dataPP_DOP = None
368 dataPP_DOP = None
356 dataPP_WIDTH = None
369 dataPP_WIDTH = None
357 dataPP_SNR = None
370 dataPP_SNR = None
358
371
359 def __init__(self):
372 def __init__(self):
360 '''
373 '''
361 Constructor
374 Constructor
362 '''
375 '''
363
376
364 self.useLocalTime = True
377 self.useLocalTime = True
365 self.radarControllerHeaderObj = RadarControllerHeader()
378 self.radarControllerHeaderObj = RadarControllerHeader()
366 self.systemHeaderObj = SystemHeader()
379 self.systemHeaderObj = SystemHeader()
380 self.processingHeaderObj = ProcessingHeader()
367 self.type = "Voltage"
381 self.type = "Voltage"
368 self.data = None
382 self.data = None
369 self.nProfiles = None
383 self.nProfiles = None
370 self.heightList = None
384 self.heightList = None
371 self.channelList = None
385 self.channelList = None
372 self.flagNoData = True
386 self.flagNoData = True
373 self.flagDiscontinuousBlock = False
387 self.flagDiscontinuousBlock = False
374 self.utctime = None
388 self.utctime = None
375 self.timeZone = 0
389 self.timeZone = 0
376 self.dstFlag = None
390 self.dstFlag = None
377 self.errorCount = None
391 self.errorCount = None
378 self.nCohInt = None
392 self.nCohInt = None
379 self.blocksize = None
393 self.blocksize = None
380 self.flagCohInt = False
394 self.flagCohInt = False
381 self.flagDecodeData = False # asumo q la data no esta decodificada
395 self.flagDecodeData = False # asumo q la data no esta decodificada
382 self.flagDeflipData = False # asumo q la data no esta sin flip
396 self.flagDeflipData = False # asumo q la data no esta sin flip
383 self.flagShiftFFT = False
397 self.flagShiftFFT = False
384 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
398 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
385 self.profileIndex = 0
399 self.profileIndex = 0
386 self.metadata_list = ['type', 'heightList', 'timeZone', 'nProfiles', 'channelList', 'nCohInt',
400 self.metadata_list = ['type', 'heightList', 'timeZone', 'nProfiles', 'channelList', 'nCohInt',
387 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp']
401 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp']
388
402
389 def getNoisebyHildebrand(self, channel=None, ymin_index=None, ymax_index=None):
403 def getNoisebyHildebrand(self, channel=None, ymin_index=None, ymax_index=None):
390 """
404 """
391 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
405 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
392
406
393 Return:
407 Return:
394 noiselevel
408 noiselevel
395 """
409 """
396
410
397 if channel != None:
411 if channel != None:
398 data = self.data[channel,ymin_index:ymax_index]
412 data = self.data[channel,ymin_index:ymax_index]
399 nChannels = 1
413 nChannels = 1
400 else:
414 else:
401 data = self.data[:,ymin_index:ymax_index]
415 data = self.data[:,ymin_index:ymax_index]
402 nChannels = self.nChannels
416 nChannels = self.nChannels
403
417
404 noise = numpy.zeros(nChannels)
418 noise = numpy.zeros(nChannels)
405 power = data * numpy.conjugate(data)
419 power = data * numpy.conjugate(data)
406
420
407 for thisChannel in range(nChannels):
421 for thisChannel in range(nChannels):
408 if nChannels == 1:
422 if nChannels == 1:
409 daux = power[:].real
423 daux = power[:].real
410 else:
424 else:
411 daux = power[thisChannel, :].real
425 daux = power[thisChannel, :].real
412 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
426 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
413
427
414 return noise
428 return noise
415
429
416 def getNoise(self, type=1, channel=None,ymin_index=None, ymax_index=None):
430 def getNoise(self, type=1, channel=None,ymin_index=None, ymax_index=None):
417
431
418 if type == 1:
432 if type == 1:
419 noise = self.getNoisebyHildebrand(channel,ymin_index, ymax_index)
433 noise = self.getNoisebyHildebrand(channel,ymin_index, ymax_index)
420
434
421 return noise
435 return noise
422
436
423 def getPower(self, channel=None):
437 def getPower(self, channel=None):
424
438
425 if channel != None:
439 if channel != None:
426 data = self.data[channel]
440 data = self.data[channel]
427 else:
441 else:
428 data = self.data
442 data = self.data
429
443
430 power = data * numpy.conjugate(data)
444 power = data * numpy.conjugate(data)
431 powerdB = 10 * numpy.log10(power.real)
445 powerdB = 10 * numpy.log10(power.real)
432 powerdB = numpy.squeeze(powerdB)
446 powerdB = numpy.squeeze(powerdB)
433
447
434 return powerdB
448 return powerdB
435
449
436 @property
450 @property
437 def timeInterval(self):
451 def timeInterval(self):
438
452
439 return self.ippSeconds * self.nCohInt
453 return self.ippSeconds * self.nCohInt
440
454
441 noise = property(getNoise, "I'm the 'nHeights' property.")
455 noise = property(getNoise, "I'm the 'nHeights' property.")
442
456
443
457
444 class Spectra(JROData):
458 class Spectra(JROData):
445
459
446 data_outlier = None
460 data_outlier = None
447
461
448 def __init__(self):
462 def __init__(self):
449 '''
463 '''
450 Constructor
464 Constructor
451 '''
465 '''
452
466
453 self.data_dc = None
467 self.data_dc = None
454 self.data_spc = None
468 self.data_spc = None
455 self.data_cspc = None
469 self.data_cspc = None
456 self.useLocalTime = True
470 self.useLocalTime = True
457 self.radarControllerHeaderObj = RadarControllerHeader()
471 self.radarControllerHeaderObj = RadarControllerHeader()
458 self.systemHeaderObj = SystemHeader()
472 self.systemHeaderObj = SystemHeader()
473 self.processingHeaderObj = ProcessingHeader()
459 self.type = "Spectra"
474 self.type = "Spectra"
460 self.timeZone = 0
475 self.timeZone = 0
461 self.nProfiles = None
476 self.nProfiles = None
462 self.heightList = None
477 self.heightList = None
463 self.channelList = None
478 self.channelList = None
464 self.pairsList = None
479 self.pairsList = None
465 self.flagNoData = True
480 self.flagNoData = True
466 self.flagDiscontinuousBlock = False
481 self.flagDiscontinuousBlock = False
467 self.utctime = None
482 self.utctime = None
468 self.nCohInt = None
483 self.nCohInt = None
469 self.nIncohInt = None
484 self.nIncohInt = None
470 self.blocksize = None
485 self.blocksize = None
471 self.nFFTPoints = None
486 self.nFFTPoints = None
472 self.wavelength = None
487 self.wavelength = None
473 self.flagDecodeData = False # asumo q la data no esta decodificada
488 self.flagDecodeData = False # asumo q la data no esta decodificada
474 self.flagDeflipData = False # asumo q la data no esta sin flip
489 self.flagDeflipData = False # asumo q la data no esta sin flip
475 self.flagShiftFFT = False
490 self.flagShiftFFT = False
476 self.ippFactor = 1
491 self.ippFactor = 1
477 self.beacon_heiIndexList = []
492 self.beacon_heiIndexList = []
478 self.noise_estimation = None
493 self.noise_estimation = None
479 self.codeList = []
494 self.codeList = []
480 self.azimuthList = []
495 self.azimuthList = []
481 self.elevationList = []
496 self.elevationList = []
482 self.metadata_list = ['type', 'heightList', 'timeZone', 'pairsList', 'channelList', 'nCohInt',
497 self.metadata_list = ['type', 'heightList', 'timeZone', 'pairsList', 'channelList', 'nCohInt',
483 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp','nIncohInt', 'nFFTPoints', 'nProfiles']
498 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp','nIncohInt', 'nFFTPoints', 'nProfiles']
484
499
485
500
486
501
487
502
488 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
503 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
489 """
504 """
490 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
505 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
491
506
492 Return:
507 Return:
493 noiselevel
508 noiselevel
494 """
509 """
495 # if hasattr(self.nIncohInt, "__len__"): #nIncohInt is a matrix
510 # if hasattr(self.nIncohInt, "__len__"): #nIncohInt is a matrix
496 #
511 #
497 # heis = self.data_spc.shape[2]
512 # heis = self.data_spc.shape[2]
498 #
513 #
499 # noise = numpy.zeros((self.nChannels, heis))
514 # noise = numpy.zeros((self.nChannels, heis))
500 # for hei in range(heis):
515 # for hei in range(heis):
501 # for channel in range(self.nChannels):
516 # for channel in range(self.nChannels):
502 # daux = self.data_spc[channel, xmin_index:xmax_index, hei]
517 # daux = self.data_spc[channel, xmin_index:xmax_index, hei]
503 #
518 #
504 # noise[channel,hei] = hildebrand_sekhon(daux, self.nIncohInt[channel,hei])
519 # noise[channel,hei] = hildebrand_sekhon(daux, self.nIncohInt[channel,hei])
505 #
520 #
506 # else:
521 # else:
507 # noise = numpy.zeros(self.nChannels)
522 # noise = numpy.zeros(self.nChannels)
508 # for channel in range(self.nChannels):
523 # for channel in range(self.nChannels):
509 # daux = self.data_spc[channel,xmin_index:xmax_index, ymin_index:ymax_index]
524 # daux = self.data_spc[channel,xmin_index:xmax_index, ymin_index:ymax_index]
510 #
525 #
511 # noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
526 # noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
512 noise = numpy.zeros(self.nChannels)
527 noise = numpy.zeros(self.nChannels)
513
528
514 for channel in range(self.nChannels):
529 for channel in range(self.nChannels):
515 daux = self.data_spc[channel,xmin_index:xmax_index, ymin_index:ymax_index]
530 daux = self.data_spc[channel,xmin_index:xmax_index, ymin_index:ymax_index]
516
531
517 noise[channel] = hildebrand_sekhon(daux, self.max_nIncohInt[channel])
532 noise[channel] = hildebrand_sekhon(daux, self.max_nIncohInt[channel])
518
533
519 return noise
534 return noise
520
535
521 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
536 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
522
537
523 if self.noise_estimation is not None:
538 if self.noise_estimation is not None:
524 # this was estimated by getNoise Operation defined in jroproc_spectra.py
539 # this was estimated by getNoise Operation defined in jroproc_spectra.py
525 return self.noise_estimation
540 return self.noise_estimation
526 else:
541 else:
527 noise = self.getNoisebyHildebrand(
542 noise = self.getNoisebyHildebrand(
528 xmin_index, xmax_index, ymin_index, ymax_index)
543 xmin_index, xmax_index, ymin_index, ymax_index)
529 return noise
544 return noise
530
545
531 def getFreqRangeTimeResponse(self, extrapoints=0):
546 def getFreqRangeTimeResponse(self, extrapoints=0):
532
547
533 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
548 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
534 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.) - deltafreq / 2
549 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.) - deltafreq / 2
535
550
536 return freqrange
551 return freqrange
537
552
538 def getAcfRange(self, extrapoints=0):
553 def getAcfRange(self, extrapoints=0):
539
554
540 deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
555 deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
541 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
556 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
542
557
543 return freqrange
558 return freqrange
544
559
545 def getFreqRange(self, extrapoints=0):
560 def getFreqRange(self, extrapoints=0):
546
561
547 deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
562 deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
548 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
563 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
549
564
550 return freqrange
565 return freqrange
551
566
552 def getVelRange(self, extrapoints=0):
567 def getVelRange(self, extrapoints=0):
553
568
554 deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
569 deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
555 velrange = deltav * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.)
570 velrange = deltav * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.)
556
571
557 if self.nmodes:
572 if self.nmodes:
558 return velrange/self.nmodes
573 return velrange/self.nmodes
559 else:
574 else:
560 return velrange
575 return velrange
561
576
562 @property
577 @property
563 def nPairs(self):
578 def nPairs(self):
564
579
565 return len(self.pairsList)
580 return len(self.pairsList)
566
581
567 @property
582 @property
568 def pairsIndexList(self):
583 def pairsIndexList(self):
569
584
570 return list(range(self.nPairs))
585 return list(range(self.nPairs))
571
586
572 @property
587 @property
573 def normFactor(self):
588 def normFactor(self):
574
589
575 pwcode = 1
590 pwcode = 1
576
591
577 if self.flagDecodeData:
592 if self.flagDecodeData:
578 pwcode = numpy.sum(self.code[0]**2)
593 pwcode = numpy.sum(self.code[0]**2)
579 #print(self.flagDecodeData, pwcode)
594 #print(self.flagDecodeData, pwcode)
580 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
595 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
581 normFactor = self.nProfiles * self.nIncohInt * self.nCohInt * pwcode * self.windowOfFilter
596 normFactor = self.nProfiles * self.nIncohInt * self.nCohInt * pwcode * self.windowOfFilter
582
597
583
598
584 return normFactor
599 return normFactor
585
600
586 @property
601 @property
587 def flag_cspc(self):
602 def flag_cspc(self):
588
603
589 if self.data_cspc is None:
604 if self.data_cspc is None:
590 return True
605 return True
591
606
592 return False
607 return False
593
608
594 @property
609 @property
595 def flag_dc(self):
610 def flag_dc(self):
596
611
597 if self.data_dc is None:
612 if self.data_dc is None:
598 return True
613 return True
599
614
600 return False
615 return False
601
616
602 @property
617 @property
603 def timeInterval(self):
618 def timeInterval(self):
604
619
605 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
620 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
606 if self.nmodes:
621 if self.nmodes:
607 return self.nmodes*timeInterval
622 return self.nmodes*timeInterval
608 else:
623 else:
609 return timeInterval
624 return timeInterval
610
625
611 def getPower(self):
626 def getPower(self):
612
627
613 factor = self.normFactor
628 factor = self.normFactor
614 power = numpy.zeros( (self.nChannels,self.nHeights) )
629 power = numpy.zeros( (self.nChannels,self.nHeights) )
615 for ch in range(self.nChannels):
630 for ch in range(self.nChannels):
616 z = None
631 z = None
617 if hasattr(factor,'shape'):
632 if hasattr(factor,'shape'):
618 if factor.ndim > 1:
633 if factor.ndim > 1:
619 z = self.data_spc[ch]/factor[ch]
634 z = self.data_spc[ch]/factor[ch]
620 else:
635 else:
621 z = self.data_spc[ch]/factor
636 z = self.data_spc[ch]/factor
622 else:
637 else:
623 z = self.data_spc[ch]/factor
638 z = self.data_spc[ch]/factor
624 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
639 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
625 avg = numpy.average(z, axis=0)
640 avg = numpy.average(z, axis=0)
626 power[ch] = 10 * numpy.log10(avg)
641 power[ch] = 10 * numpy.log10(avg)
627 return power
642 return power
628
643
629 @property
644 @property
630 def max_nIncohInt(self):
645 def max_nIncohInt(self):
631
646
632 ints = numpy.zeros(self.nChannels)
647 ints = numpy.zeros(self.nChannels)
633 for ch in range(self.nChannels):
648 for ch in range(self.nChannels):
634 if hasattr(self.nIncohInt,'shape'):
649 if hasattr(self.nIncohInt,'shape'):
635 if self.nIncohInt.ndim > 1:
650 if self.nIncohInt.ndim > 1:
636 ints[ch,] = self.nIncohInt[ch].max()
651 ints[ch,] = self.nIncohInt[ch].max()
637 else:
652 else:
638 ints[ch,] = self.nIncohInt
653 ints[ch,] = self.nIncohInt
639 self.nIncohInt = int(self.nIncohInt)
654 self.nIncohInt = int(self.nIncohInt)
640 else:
655 else:
641 ints[ch,] = self.nIncohInt
656 ints[ch,] = self.nIncohInt
642
657
643 return ints
658 return ints
644
659
645
660
646 def getCoherence(self, pairsList=None, phase=False):
661 def getCoherence(self, pairsList=None, phase=False):
647
662
648 z = []
663 z = []
649 if pairsList is None:
664 if pairsList is None:
650 pairsIndexList = self.pairsIndexList
665 pairsIndexList = self.pairsIndexList
651 else:
666 else:
652 pairsIndexList = []
667 pairsIndexList = []
653 for pair in pairsList:
668 for pair in pairsList:
654 if pair not in self.pairsList:
669 if pair not in self.pairsList:
655 raise ValueError("Pair %s is not in dataOut.pairsList" % (
670 raise ValueError("Pair %s is not in dataOut.pairsList" % (
656 pair))
671 pair))
657 pairsIndexList.append(self.pairsList.index(pair))
672 pairsIndexList.append(self.pairsList.index(pair))
658 for i in range(len(pairsIndexList)):
673 for i in range(len(pairsIndexList)):
659 pair = self.pairsList[pairsIndexList[i]]
674 pair = self.pairsList[pairsIndexList[i]]
660 ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
675 ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
661 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
676 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
662 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
677 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
663 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
678 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
664 if phase:
679 if phase:
665 data = numpy.arctan2(avgcoherenceComplex.imag,
680 data = numpy.arctan2(avgcoherenceComplex.imag,
666 avgcoherenceComplex.real) * 180 / numpy.pi
681 avgcoherenceComplex.real) * 180 / numpy.pi
667 else:
682 else:
668 data = numpy.abs(avgcoherenceComplex)
683 data = numpy.abs(avgcoherenceComplex)
669
684
670 z.append(data)
685 z.append(data)
671
686
672 return numpy.array(z)
687 return numpy.array(z)
673
688
674 def setValue(self, value):
689 def setValue(self, value):
675
690
676 print("This property should not be initialized", value)
691 print("This property should not be initialized", value)
677
692
678 return
693 return
679
694
680 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
695 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
681
696
682
697
683 class SpectraHeis(Spectra):
698 class SpectraHeis(Spectra):
684
699
685 def __init__(self):
700 def __init__(self):
686
701
687 self.radarControllerHeaderObj = RadarControllerHeader()
702 self.radarControllerHeaderObj = RadarControllerHeader()
688 self.systemHeaderObj = SystemHeader()
703 self.systemHeaderObj = SystemHeader()
689 self.type = "SpectraHeis"
704 self.type = "SpectraHeis"
690 self.nProfiles = None
705 self.nProfiles = None
691 self.heightList = None
706 self.heightList = None
692 self.channelList = None
707 self.channelList = None
693 self.flagNoData = True
708 self.flagNoData = True
694 self.flagDiscontinuousBlock = False
709 self.flagDiscontinuousBlock = False
695 self.utctime = None
710 self.utctime = None
696 self.blocksize = None
711 self.blocksize = None
697 self.profileIndex = 0
712 self.profileIndex = 0
698 self.nCohInt = 1
713 self.nCohInt = 1
699 self.nIncohInt = 1
714 self.nIncohInt = 1
700
715
701 @property
716 @property
702 def normFactor(self):
717 def normFactor(self):
703 pwcode = 1
718 pwcode = 1
704 if self.flagDecodeData:
719 if self.flagDecodeData:
705 pwcode = numpy.sum(self.code[0]**2)
720 pwcode = numpy.sum(self.code[0]**2)
706
721
707 normFactor = self.nIncohInt * self.nCohInt * pwcode
722 normFactor = self.nIncohInt * self.nCohInt * pwcode
708
723
709 return normFactor
724 return normFactor
710
725
711 @property
726 @property
712 def timeInterval(self):
727 def timeInterval(self):
713
728
714 return self.ippSeconds * self.nCohInt * self.nIncohInt
729 return self.ippSeconds * self.nCohInt * self.nIncohInt
715
730
716
731
717 class Fits(JROData):
732 class Fits(JROData):
718
733
719 def __init__(self):
734 def __init__(self):
720
735
721 self.type = "Fits"
736 self.type = "Fits"
722 self.nProfiles = None
737 self.nProfiles = None
723 self.heightList = None
738 self.heightList = None
724 self.channelList = None
739 self.channelList = None
725 self.flagNoData = True
740 self.flagNoData = True
726 self.utctime = None
741 self.utctime = None
727 self.nCohInt = 1
742 self.nCohInt = 1
728 self.nIncohInt = 1
743 self.nIncohInt = 1
729 self.useLocalTime = True
744 self.useLocalTime = True
730 self.profileIndex = 0
745 self.profileIndex = 0
731 self.timeZone = 0
746 self.timeZone = 0
732
747
733 def getTimeRange(self):
748 def getTimeRange(self):
734
749
735 datatime = []
750 datatime = []
736
751
737 datatime.append(self.ltctime)
752 datatime.append(self.ltctime)
738 datatime.append(self.ltctime + self.timeInterval)
753 datatime.append(self.ltctime + self.timeInterval)
739
754
740 datatime = numpy.array(datatime)
755 datatime = numpy.array(datatime)
741
756
742 return datatime
757 return datatime
743
758
744 def getChannelIndexList(self):
759 def getChannelIndexList(self):
745
760
746 return list(range(self.nChannels))
761 return list(range(self.nChannels))
747
762
748 def getNoise(self, type=1):
763 def getNoise(self, type=1):
749
764
750
765
751 if type == 1:
766 if type == 1:
752 noise = self.getNoisebyHildebrand()
767 noise = self.getNoisebyHildebrand()
753
768
754 if type == 2:
769 if type == 2:
755 noise = self.getNoisebySort()
770 noise = self.getNoisebySort()
756
771
757 if type == 3:
772 if type == 3:
758 noise = self.getNoisebyWindow()
773 noise = self.getNoisebyWindow()
759
774
760 return noise
775 return noise
761
776
762 @property
777 @property
763 def timeInterval(self):
778 def timeInterval(self):
764
779
765 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
780 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
766
781
767 return timeInterval
782 return timeInterval
768
783
769 @property
784 @property
770 def ippSeconds(self):
785 def ippSeconds(self):
771 '''
786 '''
772 '''
787 '''
773 return self.ipp_sec
788 return self.ipp_sec
774
789
775 noise = property(getNoise, "I'm the 'nHeights' property.")
790 noise = property(getNoise, "I'm the 'nHeights' property.")
776
791
777
792
778 class Correlation(JROData):
793 class Correlation(JROData):
779
794
780 def __init__(self):
795 def __init__(self):
781 '''
796 '''
782 Constructor
797 Constructor
783 '''
798 '''
784 self.radarControllerHeaderObj = RadarControllerHeader()
799 self.radarControllerHeaderObj = RadarControllerHeader()
785 self.systemHeaderObj = SystemHeader()
800 self.systemHeaderObj = SystemHeader()
786 self.type = "Correlation"
801 self.type = "Correlation"
787 self.data = None
802 self.data = None
788 self.dtype = None
803 self.dtype = None
789 self.nProfiles = None
804 self.nProfiles = None
790 self.heightList = None
805 self.heightList = None
791 self.channelList = None
806 self.channelList = None
792 self.flagNoData = True
807 self.flagNoData = True
793 self.flagDiscontinuousBlock = False
808 self.flagDiscontinuousBlock = False
794 self.utctime = None
809 self.utctime = None
795 self.timeZone = 0
810 self.timeZone = 0
796 self.dstFlag = None
811 self.dstFlag = None
797 self.errorCount = None
812 self.errorCount = None
798 self.blocksize = None
813 self.blocksize = None
799 self.flagDecodeData = False # asumo q la data no esta decodificada
814 self.flagDecodeData = False # asumo q la data no esta decodificada
800 self.flagDeflipData = False # asumo q la data no esta sin flip
815 self.flagDeflipData = False # asumo q la data no esta sin flip
801 self.pairsList = None
816 self.pairsList = None
802 self.nPoints = None
817 self.nPoints = None
803
818
804 def getPairsList(self):
819 def getPairsList(self):
805
820
806 return self.pairsList
821 return self.pairsList
807
822
808 def getNoise(self, mode=2):
823 def getNoise(self, mode=2):
809
824
810 indR = numpy.where(self.lagR == 0)[0][0]
825 indR = numpy.where(self.lagR == 0)[0][0]
811 indT = numpy.where(self.lagT == 0)[0][0]
826 indT = numpy.where(self.lagT == 0)[0][0]
812
827
813 jspectra0 = self.data_corr[:, :, indR, :]
828 jspectra0 = self.data_corr[:, :, indR, :]
814 jspectra = copy.copy(jspectra0)
829 jspectra = copy.copy(jspectra0)
815
830
816 num_chan = jspectra.shape[0]
831 num_chan = jspectra.shape[0]
817 num_hei = jspectra.shape[2]
832 num_hei = jspectra.shape[2]
818
833
819 freq_dc = jspectra.shape[1] / 2
834 freq_dc = jspectra.shape[1] / 2
820 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
835 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
821
836
822 if ind_vel[0] < 0:
837 if ind_vel[0] < 0:
823 ind_vel[list(range(0, 1))] = ind_vel[list(
838 ind_vel[list(range(0, 1))] = ind_vel[list(
824 range(0, 1))] + self.num_prof
839 range(0, 1))] + self.num_prof
825
840
826 if mode == 1:
841 if mode == 1:
827 jspectra[:, freq_dc, :] = (
842 jspectra[:, freq_dc, :] = (
828 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
843 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
829
844
830 if mode == 2:
845 if mode == 2:
831
846
832 vel = numpy.array([-2, -1, 1, 2])
847 vel = numpy.array([-2, -1, 1, 2])
833 xx = numpy.zeros([4, 4])
848 xx = numpy.zeros([4, 4])
834
849
835 for fil in range(4):
850 for fil in range(4):
836 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
851 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
837
852
838 xx_inv = numpy.linalg.inv(xx)
853 xx_inv = numpy.linalg.inv(xx)
839 xx_aux = xx_inv[0, :]
854 xx_aux = xx_inv[0, :]
840
855
841 for ich in range(num_chan):
856 for ich in range(num_chan):
842 yy = jspectra[ich, ind_vel, :]
857 yy = jspectra[ich, ind_vel, :]
843 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
858 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
844
859
845 junkid = jspectra[ich, freq_dc, :] <= 0
860 junkid = jspectra[ich, freq_dc, :] <= 0
846 cjunkid = sum(junkid)
861 cjunkid = sum(junkid)
847
862
848 if cjunkid.any():
863 if cjunkid.any():
849 jspectra[ich, freq_dc, junkid.nonzero()] = (
864 jspectra[ich, freq_dc, junkid.nonzero()] = (
850 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
865 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
851
866
852 noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
867 noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
853
868
854 return noise
869 return noise
855
870
856 @property
871 @property
857 def timeInterval(self):
872 def timeInterval(self):
858
873
859 return self.ippSeconds * self.nCohInt * self.nProfiles
874 return self.ippSeconds * self.nCohInt * self.nProfiles
860
875
861 def splitFunctions(self):
876 def splitFunctions(self):
862
877
863 pairsList = self.pairsList
878 pairsList = self.pairsList
864 ccf_pairs = []
879 ccf_pairs = []
865 acf_pairs = []
880 acf_pairs = []
866 ccf_ind = []
881 ccf_ind = []
867 acf_ind = []
882 acf_ind = []
868 for l in range(len(pairsList)):
883 for l in range(len(pairsList)):
869 chan0 = pairsList[l][0]
884 chan0 = pairsList[l][0]
870 chan1 = pairsList[l][1]
885 chan1 = pairsList[l][1]
871
886
872 # Obteniendo pares de Autocorrelacion
887 # Obteniendo pares de Autocorrelacion
873 if chan0 == chan1:
888 if chan0 == chan1:
874 acf_pairs.append(chan0)
889 acf_pairs.append(chan0)
875 acf_ind.append(l)
890 acf_ind.append(l)
876 else:
891 else:
877 ccf_pairs.append(pairsList[l])
892 ccf_pairs.append(pairsList[l])
878 ccf_ind.append(l)
893 ccf_ind.append(l)
879
894
880 data_acf = self.data_cf[acf_ind]
895 data_acf = self.data_cf[acf_ind]
881 data_ccf = self.data_cf[ccf_ind]
896 data_ccf = self.data_cf[ccf_ind]
882
897
883 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
898 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
884
899
885 @property
900 @property
886 def normFactor(self):
901 def normFactor(self):
887 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
902 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
888 acf_pairs = numpy.array(acf_pairs)
903 acf_pairs = numpy.array(acf_pairs)
889 normFactor = numpy.zeros((self.nPairs, self.nHeights))
904 normFactor = numpy.zeros((self.nPairs, self.nHeights))
890
905
891 for p in range(self.nPairs):
906 for p in range(self.nPairs):
892 pair = self.pairsList[p]
907 pair = self.pairsList[p]
893
908
894 ch0 = pair[0]
909 ch0 = pair[0]
895 ch1 = pair[1]
910 ch1 = pair[1]
896
911
897 ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
912 ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
898 ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
913 ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
899 normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
914 normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
900
915
901 return normFactor
916 return normFactor
902
917
903
918
904 class Parameters(Spectra):
919 class Parameters(Spectra):
905
920
906 radarControllerHeaderTxt=None #header Controller like text
921 radarControllerHeaderTxt=None #header Controller like text
907 groupList = None # List of Pairs, Groups, etc
922 groupList = None # List of Pairs, Groups, etc
908 data_param = None # Parameters obtained
923 data_param = None # Parameters obtained
909 data_pre = None # Data Pre Parametrization
924 data_pre = None # Data Pre Parametrization
910 data_SNR = None # Signal to Noise Ratio
925 data_SNR = None # Signal to Noise Ratio
911 data_outlier = None
926 data_outlier = None
912 abscissaList = None # Abscissa, can be velocities, lags or time
927 abscissaList = None # Abscissa, can be velocities, lags or time
913 utctimeInit = None # Initial UTC time
928 utctimeInit = None # Initial UTC time
914 paramInterval = None # Time interval to calculate Parameters in seconds
929 paramInterval = None # Time interval to calculate Parameters in seconds
915 useLocalTime = True
930 useLocalTime = True
916 # Fitting
931 # Fitting
917 data_error = None # Error of the estimation
932 data_error = None # Error of the estimation
918 constants = None
933 constants = None
919 library = None
934 library = None
920 # Output signal
935 # Output signal
921 outputInterval = None # Time interval to calculate output signal in seconds
936 outputInterval = None # Time interval to calculate output signal in seconds
922 data_output = None # Out signal
937 data_output = None # Out signal
923 nAvg = None
938 nAvg = None
924 noise_estimation = None
939 noise_estimation = None
925 GauSPC = None # Fit gaussian SPC
940 GauSPC = None # Fit gaussian SPC
926
941
942
943
927 def __init__(self):
944 def __init__(self):
928 '''
945 '''
929 Constructor
946 Constructor
930 '''
947 '''
931 self.radarControllerHeaderObj = RadarControllerHeader()
948 self.radarControllerHeaderObj = RadarControllerHeader()
932 self.systemHeaderObj = SystemHeader()
949 self.systemHeaderObj = SystemHeader()
950 self.processingHeaderObj = ProcessingHeader()
933 self.type = "Parameters"
951 self.type = "Parameters"
934 self.timeZone = 0
952 self.timeZone = 0
935
953
936 def getTimeRange1(self, interval):
954 def getTimeRange1(self, interval):
937
955
938 datatime = []
956 datatime = []
939
957
940 if self.useLocalTime:
958 if self.useLocalTime:
941 time1 = self.utctimeInit - self.timeZone * 60
959 time1 = self.utctimeInit - self.timeZone * 60
942 else:
960 else:
943 time1 = self.utctimeInit
961 time1 = self.utctimeInit
944
962
945 datatime.append(time1)
963 datatime.append(time1)
946 datatime.append(time1 + interval)
964 datatime.append(time1 + interval)
947 datatime = numpy.array(datatime)
965 datatime = numpy.array(datatime)
948
966
949 return datatime
967 return datatime
950
968
951 @property
969 @property
952 def timeInterval(self):
970 def timeInterval(self):
953
971
954 if hasattr(self, 'timeInterval1'):
972 if hasattr(self, 'timeInterval1'):
955 return self.timeInterval1
973 return self.timeInterval1
956 else:
974 else:
957 return self.paramInterval
975 return self.paramInterval
958
976
959 def setValue(self, value):
977 def setValue(self, value):
960
978
961 print("This property should not be initialized")
979 print("This property should not be initialized")
962
980
963 return
981 return
964
982
965 def getNoise(self):
983 def getNoise(self):
966
984
967 return self.spc_noise
985 return self.spc_noise
968
986
969 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
987 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
970
988
971
989
972 class PlotterData(object):
990 class PlotterData(object):
973 '''
991 '''
974 Object to hold data to be plotted
992 Object to hold data to be plotted
975 '''
993 '''
976
994
977 MAXNUMX = 200
995 MAXNUMX = 200
978 MAXNUMY = 200
996 MAXNUMY = 200
979
997
980 def __init__(self, code, exp_code, localtime=True):
998 def __init__(self, code, exp_code, localtime=True):
981
999
982 self.key = code
1000 self.key = code
983 self.exp_code = exp_code
1001 self.exp_code = exp_code
984 self.ready = False
1002 self.ready = False
985 self.flagNoData = False
1003 self.flagNoData = False
986 self.localtime = localtime
1004 self.localtime = localtime
987 self.data = {}
1005 self.data = {}
988 self.meta = {}
1006 self.meta = {}
989 self.__heights = []
1007 self.__heights = []
990
1008
991 def __str__(self):
1009 def __str__(self):
992 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
1010 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
993 return 'Data[{}][{}]'.format(';'.join(dum), len(self.times))
1011 return 'Data[{}][{}]'.format(';'.join(dum), len(self.times))
994
1012
995 def __len__(self):
1013 def __len__(self):
996 return len(self.data)
1014 return len(self.data)
997
1015
998 def __getitem__(self, key):
1016 def __getitem__(self, key):
999 if isinstance(key, int):
1017 if isinstance(key, int):
1000 return self.data[self.times[key]]
1018 return self.data[self.times[key]]
1001 elif isinstance(key, str):
1019 elif isinstance(key, str):
1002 ret = numpy.array([self.data[x][key] for x in self.times])
1020 ret = numpy.array([self.data[x][key] for x in self.times])
1003 if ret.ndim > 1:
1021 if ret.ndim > 1:
1004 ret = numpy.swapaxes(ret, 0, 1)
1022 ret = numpy.swapaxes(ret, 0, 1)
1005 return ret
1023 return ret
1006
1024
1007 def __contains__(self, key):
1025 def __contains__(self, key):
1008 return key in self.data[self.min_time]
1026 return key in self.data[self.min_time]
1009
1027
1010 def setup(self):
1028 def setup(self):
1011 '''
1029 '''
1012 Configure object
1030 Configure object
1013 '''
1031 '''
1014 self.type = ''
1032 self.type = ''
1015 self.ready = False
1033 self.ready = False
1016 del self.data
1034 del self.data
1017 self.data = {}
1035 self.data = {}
1018 self.__heights = []
1036 self.__heights = []
1019 self.__all_heights = set()
1037 self.__all_heights = set()
1020
1038
1021 def shape(self, key):
1039 def shape(self, key):
1022 '''
1040 '''
1023 Get the shape of the one-element data for the given key
1041 Get the shape of the one-element data for the given key
1024 '''
1042 '''
1025
1043
1026 if len(self.data[self.min_time][key]):
1044 if len(self.data[self.min_time][key]):
1027 return self.data[self.min_time][key].shape
1045 return self.data[self.min_time][key].shape
1028 return (0,)
1046 return (0,)
1029
1047
1030 def update(self, data, tm, meta={}):
1048 def update(self, data, tm, meta={}):
1031 '''
1049 '''
1032 Update data object with new dataOut
1050 Update data object with new dataOut
1033 '''
1051 '''
1034
1052
1035 self.data[tm] = data
1053 self.data[tm] = data
1036
1054
1037 for key, value in meta.items():
1055 for key, value in meta.items():
1038 setattr(self, key, value)
1056 setattr(self, key, value)
1039
1057
1040 def normalize_heights(self):
1058 def normalize_heights(self):
1041 '''
1059 '''
1042 Ensure same-dimension of the data for different heighList
1060 Ensure same-dimension of the data for different heighList
1043 '''
1061 '''
1044
1062
1045 H = numpy.array(list(self.__all_heights))
1063 H = numpy.array(list(self.__all_heights))
1046 H.sort()
1064 H.sort()
1047 for key in self.data:
1065 for key in self.data:
1048 shape = self.shape(key)[:-1] + H.shape
1066 shape = self.shape(key)[:-1] + H.shape
1049 for tm, obj in list(self.data[key].items()):
1067 for tm, obj in list(self.data[key].items()):
1050 h = self.__heights[self.times.tolist().index(tm)]
1068 h = self.__heights[self.times.tolist().index(tm)]
1051 if H.size == h.size:
1069 if H.size == h.size:
1052 continue
1070 continue
1053 index = numpy.where(numpy.in1d(H, h))[0]
1071 index = numpy.where(numpy.in1d(H, h))[0]
1054 dummy = numpy.zeros(shape) + numpy.nan
1072 dummy = numpy.zeros(shape) + numpy.nan
1055 if len(shape) == 2:
1073 if len(shape) == 2:
1056 dummy[:, index] = obj
1074 dummy[:, index] = obj
1057 else:
1075 else:
1058 dummy[index] = obj
1076 dummy[index] = obj
1059 self.data[key][tm] = dummy
1077 self.data[key][tm] = dummy
1060
1078
1061 self.__heights = [H for tm in self.times]
1079 self.__heights = [H for tm in self.times]
1062
1080
1063 def jsonify(self, tm, plot_name, plot_type, decimate=False):
1081 def jsonify(self, tm, plot_name, plot_type, decimate=False):
1064 '''
1082 '''
1065 Convert data to json
1083 Convert data to json
1066 '''
1084 '''
1067
1085
1068 meta = {}
1086 meta = {}
1069 meta['xrange'] = []
1087 meta['xrange'] = []
1070 dy = int(len(self.yrange)/self.MAXNUMY) + 1
1088 dy = int(len(self.yrange)/self.MAXNUMY) + 1
1071 tmp = self.data[tm][self.key]
1089 tmp = self.data[tm][self.key]
1072 shape = tmp.shape
1090 shape = tmp.shape
1073 if len(shape) == 2:
1091 if len(shape) == 2:
1074 data = self.roundFloats(self.data[tm][self.key][::, ::dy].tolist())
1092 data = self.roundFloats(self.data[tm][self.key][::, ::dy].tolist())
1075 elif len(shape) == 3:
1093 elif len(shape) == 3:
1076 dx = int(self.data[tm][self.key].shape[1]/self.MAXNUMX) + 1
1094 dx = int(self.data[tm][self.key].shape[1]/self.MAXNUMX) + 1
1077 data = self.roundFloats(
1095 data = self.roundFloats(
1078 self.data[tm][self.key][::, ::dx, ::dy].tolist())
1096 self.data[tm][self.key][::, ::dx, ::dy].tolist())
1079 meta['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
1097 meta['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
1080 else:
1098 else:
1081 data = self.roundFloats(self.data[tm][self.key].tolist())
1099 data = self.roundFloats(self.data[tm][self.key].tolist())
1082
1100
1083 ret = {
1101 ret = {
1084 'plot': plot_name,
1102 'plot': plot_name,
1085 'code': self.exp_code,
1103 'code': self.exp_code,
1086 'time': float(tm),
1104 'time': float(tm),
1087 'data': data,
1105 'data': data,
1088 }
1106 }
1089 meta['type'] = plot_type
1107 meta['type'] = plot_type
1090 meta['interval'] = float(self.interval)
1108 meta['interval'] = float(self.interval)
1091 meta['localtime'] = self.localtime
1109 meta['localtime'] = self.localtime
1092 meta['yrange'] = self.roundFloats(self.yrange[::dy].tolist())
1110 meta['yrange'] = self.roundFloats(self.yrange[::dy].tolist())
1093 meta.update(self.meta)
1111 meta.update(self.meta)
1094 ret['metadata'] = meta
1112 ret['metadata'] = meta
1095 return json.dumps(ret)
1113 return json.dumps(ret)
1096
1114
1097 @property
1115 @property
1098 def times(self):
1116 def times(self):
1099 '''
1117 '''
1100 Return the list of times of the current data
1118 Return the list of times of the current data
1101 '''
1119 '''
1102
1120
1103 ret = [t for t in self.data]
1121 ret = [t for t in self.data]
1104 ret.sort()
1122 ret.sort()
1105 return numpy.array(ret)
1123 return numpy.array(ret)
1106
1124
1107 @property
1125 @property
1108 def min_time(self):
1126 def min_time(self):
1109 '''
1127 '''
1110 Return the minimun time value
1128 Return the minimun time value
1111 '''
1129 '''
1112
1130
1113 return self.times[0]
1131 return self.times[0]
1114
1132
1115 @property
1133 @property
1116 def max_time(self):
1134 def max_time(self):
1117 '''
1135 '''
1118 Return the maximun time value
1136 Return the maximun time value
1119 '''
1137 '''
1120
1138
1121 return self.times[-1]
1139 return self.times[-1]
1122
1140
1123 # @property
1141 # @property
1124 # def heights(self):
1142 # def heights(self):
1125 # '''
1143 # '''
1126 # Return the list of heights of the current data
1144 # Return the list of heights of the current data
1127 # '''
1145 # '''
1128
1146
1129 # return numpy.array(self.__heights[-1])
1147 # return numpy.array(self.__heights[-1])
1130
1148
1131 @staticmethod
1149 @staticmethod
1132 def roundFloats(obj):
1150 def roundFloats(obj):
1133 if isinstance(obj, list):
1151 if isinstance(obj, list):
1134 return list(map(PlotterData.roundFloats, obj))
1152 return list(map(PlotterData.roundFloats, obj))
1135 elif isinstance(obj, float):
1153 elif isinstance(obj, float):
1136 return round(obj, 2)
1154 return round(obj, 2)
@@ -1,915 +1,948
1 '''
1 '''
2
2
3 $Author: murco $
3 $Author: murco $
4 $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
4 $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
5 '''
5 '''
6 import sys
6 import sys
7 import numpy
7 import numpy
8 import copy
8 import copy
9 import datetime
9 import datetime
10 import inspect
10 import inspect
11 from schainpy.utils import log
11 from schainpy.utils import log
12
12
13 SPEED_OF_LIGHT = 299792458
13 SPEED_OF_LIGHT = 299792458
14 SPEED_OF_LIGHT = 3e8
14 SPEED_OF_LIGHT = 3e8
15
15
16 BASIC_STRUCTURE = numpy.dtype([
16 BASIC_STRUCTURE = numpy.dtype([
17 ('nSize', '<u4'),
17 ('nSize', '<u4'),
18 ('nVersion', '<u2'),
18 ('nVersion', '<u2'),
19 ('nDataBlockId', '<u4'),
19 ('nDataBlockId', '<u4'),
20 ('nUtime', '<u4'),
20 ('nUtime', '<u4'),
21 ('nMilsec', '<u2'),
21 ('nMilsec', '<u2'),
22 ('nTimezone', '<i2'),
22 ('nTimezone', '<i2'),
23 ('nDstflag', '<i2'),
23 ('nDstflag', '<i2'),
24 ('nErrorCount', '<u4')
24 ('nErrorCount', '<u4')
25 ])
25 ])
26
26
27 SYSTEM_STRUCTURE = numpy.dtype([
27 SYSTEM_STRUCTURE = numpy.dtype([
28 ('nSize', '<u4'),
28 ('nSize', '<u4'),
29 ('nNumSamples', '<u4'),
29 ('nNumSamples', '<u4'),
30 ('nNumProfiles', '<u4'),
30 ('nNumProfiles', '<u4'),
31 ('nNumChannels', '<u4'),
31 ('nNumChannels', '<u4'),
32 ('nADCResolution', '<u4'),
32 ('nADCResolution', '<u4'),
33 ('nPCDIOBusWidth', '<u4'),
33 ('nPCDIOBusWidth', '<u4'),
34 ])
34 ])
35
35
36 RADAR_STRUCTURE = numpy.dtype([
36 RADAR_STRUCTURE = numpy.dtype([
37 ('nSize', '<u4'),
37 ('nSize', '<u4'),
38 ('nExpType', '<u4'),
38 ('nExpType', '<u4'),
39 ('nNTx', '<u4'),
39 ('nNTx', '<u4'),
40 ('fIpp', '<f4'),
40 ('fIpp', '<f4'),
41 ('fTxA', '<f4'),
41 ('fTxA', '<f4'),
42 ('fTxB', '<f4'),
42 ('fTxB', '<f4'),
43 ('nNumWindows', '<u4'),
43 ('nNumWindows', '<u4'),
44 ('nNumTaus', '<u4'),
44 ('nNumTaus', '<u4'),
45 ('nCodeType', '<u4'),
45 ('nCodeType', '<u4'),
46 ('nLine6Function', '<u4'),
46 ('nLine6Function', '<u4'),
47 ('nLine5Function', '<u4'),
47 ('nLine5Function', '<u4'),
48 ('fClock', '<f4'),
48 ('fClock', '<f4'),
49 ('nPrePulseBefore', '<u4'),
49 ('nPrePulseBefore', '<u4'),
50 ('nPrePulseAfter', '<u4'),
50 ('nPrePulseAfter', '<u4'),
51 ('sRangeIPP', '<a20'),
51 ('sRangeIPP', '<a20'),
52 ('sRangeTxA', '<a20'),
52 ('sRangeTxA', '<a20'),
53 ('sRangeTxB', '<a20'),
53 ('sRangeTxB', '<a20'),
54 ])
54 ])
55
55
56 SAMPLING_STRUCTURE = numpy.dtype(
56 SAMPLING_STRUCTURE = numpy.dtype(
57 [('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')])
57 [('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')])
58
58
59
59
60 PROCESSING_STRUCTURE = numpy.dtype([
60 PROCESSING_STRUCTURE = numpy.dtype([
61 ('nSize', '<u4'),
61 ('nSize', '<u4'),
62 ('nDataType', '<u4'),
62 ('nDataType', '<u4'),
63 ('nSizeOfDataBlock', '<u4'),
63 ('nSizeOfDataBlock', '<u4'),
64 ('nProfilesperBlock', '<u4'),
64 ('nProfilesperBlock', '<u4'),
65 ('nDataBlocksperFile', '<u4'),
65 ('nDataBlocksperFile', '<u4'),
66 ('nNumWindows', '<u4'),
66 ('nNumWindows', '<u4'),
67 ('nProcessFlags', '<u4'),
67 ('nProcessFlags', '<u4'),
68 ('nCoherentIntegrations', '<u4'),
68 ('nCoherentIntegrations', '<u4'),
69 ('nIncoherentIntegrations', '<u4'),
69 ('nIncoherentIntegrations', '<u4'),
70 ('nTotalSpectra', '<u4')
70 ('nTotalSpectra', '<u4')
71 ])
71 ])
72
72
73
73
74 class Header(object):
74 class Header(object):
75
75
76 def __init__(self):
76 def __init__(self):
77 raise NotImplementedError
77 raise NotImplementedError
78
78
79 def copy(self):
79 def copy(self):
80 return copy.deepcopy(self)
80 return copy.deepcopy(self)
81
81
82 def read(self):
82 def read(self):
83
83
84 raise NotImplementedError
84 raise NotImplementedError
85
85
86 def write(self):
86 def write(self):
87
87
88 raise NotImplementedError
88 raise NotImplementedError
89
89
90 def getAllowedArgs(self):
90 def getAllowedArgs(self):
91 args = inspect.getargspec(self.__init__).args
91 args = inspect.getargspec(self.__init__).args
92 try:
92 try:
93 args.remove('self')
93 args.remove('self')
94 except:
94 except:
95 pass
95 pass
96 return args
96 return args
97
97
98 def getAsDict(self):
98 def getAsDict(self):
99 args = self.getAllowedArgs()
99 args = self.getAllowedArgs()
100 asDict = {}
100 asDict = {}
101 for x in args:
101 for x in args:
102 asDict[x] = self[x]
102 asDict[x] = self[x]
103 return asDict
103 return asDict
104
104
105 def __getitem__(self, name):
105 def __getitem__(self, name):
106 return getattr(self, name)
106 return getattr(self, name)
107
107
108 def printInfo(self):
108 def printInfo(self):
109
109
110 message = "#" * 50 + "\n"
110 message = "#" * 50 + "\n"
111 message += self.__class__.__name__.upper() + "\n"
111 message += self.__class__.__name__.upper() + "\n"
112 message += "#" * 50 + "\n"
112 message += "#" * 50 + "\n"
113
113
114 keyList = list(self.__dict__.keys())
114 keyList = list(self.__dict__.keys())
115 keyList.sort()
115 keyList.sort()
116
116
117 for key in keyList:
117 for key in keyList:
118 message += "%s = %s" % (key, self.__dict__[key]) + "\n"
118 message += "%s = %s" % (key, self.__dict__[key]) + "\n"
119
119
120 if "size" not in keyList:
120 if "size" not in keyList:
121 attr = getattr(self, "size")
121 attr = getattr(self, "size")
122
122
123 if attr:
123 if attr:
124 message += "%s = %s" % ("size", attr) + "\n"
124 message += "%s = %s" % ("size", attr) + "\n"
125
125
126 print(message)
126 print(message)
127
127
128
128
129 class BasicHeader(Header):
129 class BasicHeader(Header):
130
130
131 size = None
131 size = None
132 version = None
132 version = None
133 dataBlock = None
133 dataBlock = None
134 utc = None
134 utc = None
135 ltc = None
135 ltc = None
136 miliSecond = None
136 miliSecond = None
137 timeZone = None
137 timeZone = None
138 dstFlag = None
138 dstFlag = None
139 errorCount = None
139 errorCount = None
140 datatime = None
140 datatime = None
141 structure = BASIC_STRUCTURE
141 structure = BASIC_STRUCTURE
142 __LOCALTIME = None
142 __LOCALTIME = None
143
143
144 def __init__(self, useLocalTime=True):
144 def __init__(self, useLocalTime=True):
145
145
146 self.size = 24
146 self.size = 24
147 self.version = 0
147 self.version = 0
148 self.dataBlock = 0
148 self.dataBlock = 0
149 self.utc = 0
149 self.utc = 0
150 self.miliSecond = 0
150 self.miliSecond = 0
151 self.timeZone = 0
151 self.timeZone = 0
152 self.dstFlag = 0
152 self.dstFlag = 0
153 self.errorCount = 0
153 self.errorCount = 0
154
154
155 self.useLocalTime = useLocalTime
155 self.useLocalTime = useLocalTime
156
156
157 def read(self, fp):
157 def read(self, fp):
158
158
159 self.length = 0
159 self.length = 0
160 try:
160 try:
161 if hasattr(fp, 'read'):
161 if hasattr(fp, 'read'):
162 header = numpy.fromfile(fp, BASIC_STRUCTURE, 1)
162 header = numpy.fromfile(fp, BASIC_STRUCTURE, 1)
163 else:
163 else:
164 header = numpy.fromstring(fp, BASIC_STRUCTURE, 1)
164 header = numpy.fromstring(fp, BASIC_STRUCTURE, 1)
165 except Exception as e:
165 except Exception as e:
166 print("BasicHeader: ")
166 print("BasicHeader: ")
167 print(e)
167 print(e)
168 return 0
168 return 0
169
169
170 self.size = int(header['nSize'][0])
170 self.size = int(header['nSize'][0])
171 self.version = int(header['nVersion'][0])
171 self.version = int(header['nVersion'][0])
172 self.dataBlock = int(header['nDataBlockId'][0])
172 self.dataBlock = int(header['nDataBlockId'][0])
173 self.utc = int(header['nUtime'][0])
173 self.utc = int(header['nUtime'][0])
174 self.miliSecond = int(header['nMilsec'][0])
174 self.miliSecond = int(header['nMilsec'][0])
175 self.timeZone = int(header['nTimezone'][0])
175 self.timeZone = int(header['nTimezone'][0])
176 self.dstFlag = int(header['nDstflag'][0])
176 self.dstFlag = int(header['nDstflag'][0])
177 self.errorCount = int(header['nErrorCount'][0])
177 self.errorCount = int(header['nErrorCount'][0])
178
178
179 if self.size < 24:
179 if self.size < 24:
180 return 0
180 return 0
181
181
182 self.length = header.nbytes
182 self.length = header.nbytes
183 return 1
183 return 1
184
184
185 def write(self, fp):
185 def write(self, fp):
186
186
187 headerTuple = (self.size, self.version, self.dataBlock, self.utc,
187 headerTuple = (self.size, self.version, self.dataBlock, self.utc,
188 self.miliSecond, self.timeZone, self.dstFlag, self.errorCount)
188 self.miliSecond, self.timeZone, self.dstFlag, self.errorCount)
189 header = numpy.array(headerTuple, BASIC_STRUCTURE)
189 header = numpy.array(headerTuple, BASIC_STRUCTURE)
190 header.tofile(fp)
190 header.tofile(fp)
191
191
192 return 1
192 return 1
193
193
194 def get_ltc(self):
194 def get_ltc(self):
195
195
196 return self.utc - self.timeZone * 60
196 return self.utc - self.timeZone * 60
197
197
198 def set_ltc(self, value):
198 def set_ltc(self, value):
199
199
200 self.utc = value + self.timeZone * 60
200 self.utc = value + self.timeZone * 60
201
201
202 def get_datatime(self):
202 def get_datatime(self):
203
203
204 return datetime.datetime.utcfromtimestamp(self.ltc)
204 return datetime.datetime.utcfromtimestamp(self.ltc)
205
205
206 ltc = property(get_ltc, set_ltc)
206 ltc = property(get_ltc, set_ltc)
207 datatime = property(get_datatime)
207 datatime = property(get_datatime)
208
208
209
209
210 class SystemHeader(Header):
210 class SystemHeader(Header):
211
211
212 size = None
212 size = None
213 nSamples = None
213 nSamples = None
214 nProfiles = None
214 nProfiles = None
215 nChannels = None
215 nChannels = None
216 adcResolution = None
216 adcResolution = None
217 pciDioBusWidth = None
217 pciDioBusWidth = None
218 structure = SYSTEM_STRUCTURE
218 structure = SYSTEM_STRUCTURE
219
219
220 def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWidth=0):
220 def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWidth=0):
221
221
222 self.size = 24
222 self.size = 24
223 self.nSamples = nSamples
223 self.nSamples = nSamples
224 self.nProfiles = nProfiles
224 self.nProfiles = nProfiles
225 self.nChannels = nChannels
225 self.nChannels = nChannels
226 self.adcResolution = adcResolution
226 self.adcResolution = adcResolution
227 self.pciDioBusWidth = pciDioBusWidth
227 self.pciDioBusWidth = pciDioBusWidth
228
228
229 def read(self, fp):
229 def read(self, fp):
230 self.length = 0
230 self.length = 0
231 try:
231 try:
232 startFp = fp.tell()
232 startFp = fp.tell()
233 except Exception as e:
233 except Exception as e:
234 startFp = None
234 startFp = None
235 pass
235 pass
236
236
237 try:
237 try:
238 if hasattr(fp, 'read'):
238 if hasattr(fp, 'read'):
239 header = numpy.fromfile(fp, SYSTEM_STRUCTURE, 1)
239 header = numpy.fromfile(fp, SYSTEM_STRUCTURE, 1)
240 else:
240 else:
241 header = numpy.fromstring(fp, SYSTEM_STRUCTURE, 1)
241 header = numpy.fromstring(fp, SYSTEM_STRUCTURE, 1)
242 except Exception as e:
242 except Exception as e:
243 print("System Header: " + str(e))
243 print("System Header: " + str(e))
244 return 0
244 return 0
245
245
246 self.size = header['nSize'][0]
246 self.size = header['nSize'][0]
247 self.nSamples = header['nNumSamples'][0]
247 self.nSamples = header['nNumSamples'][0]
248 self.nProfiles = header['nNumProfiles'][0]
248 self.nProfiles = header['nNumProfiles'][0]
249 self.nChannels = header['nNumChannels'][0]
249 self.nChannels = header['nNumChannels'][0]
250 self.adcResolution = header['nADCResolution'][0]
250 self.adcResolution = header['nADCResolution'][0]
251 self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
251 self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
252
252
253 if startFp is not None:
253 if startFp is not None:
254 endFp = self.size + startFp
254 endFp = self.size + startFp
255
255
256 if fp.tell() > endFp:
256 if fp.tell() > endFp:
257 sys.stderr.write(
257 sys.stderr.write(
258 "Warning %s: Size value read from System Header is lower than it has to be\n" % fp.name)
258 "Warning %s: Size value read from System Header is lower than it has to be\n" % fp.name)
259 return 0
259 return 0
260
260
261 if fp.tell() < endFp:
261 if fp.tell() < endFp:
262 sys.stderr.write(
262 sys.stderr.write(
263 "Warning %s: Size value read from System Header size is greater than it has to be\n" % fp.name)
263 "Warning %s: Size value read from System Header size is greater than it has to be\n" % fp.name)
264 return 0
264 return 0
265
265
266 self.length = header.nbytes
266 self.length = header.nbytes
267 return 1
267 return 1
268
268
269 def write(self, fp):
269 def write(self, fp):
270
270
271 headerTuple = (self.size, self.nSamples, self.nProfiles,
271 headerTuple = (self.size, self.nSamples, self.nProfiles,
272 self.nChannels, self.adcResolution, self.pciDioBusWidth)
272 self.nChannels, self.adcResolution, self.pciDioBusWidth)
273 header = numpy.array(headerTuple, SYSTEM_STRUCTURE)
273 header = numpy.array(headerTuple, SYSTEM_STRUCTURE)
274 header.tofile(fp)
274 header.tofile(fp)
275
275
276 return 1
276 return 1
277
277
278
278
279 class RadarControllerHeader(Header):
279 class RadarControllerHeader(Header):
280
280
281 expType = None
281 expType = None
282 dtype = ""
282 nTx = None
283 nTx = None
283 ipp = None
284 ipp = None
284 txA = None
285 txA = None
285 txB = None
286 txB = None
286 nWindows = None
287 nWindows = None
287 numTaus = None
288 numTaus = None
288 codeType = None
289 codeType = None
289 line6Function = None
290 line6Function = None
290 line5Function = None
291 line5Function = None
291 fClock = None
292 fClock = None
292 prePulseBefore = None
293 prePulseBefore = None
293 prePulseAfter = None
294 prePulseAfter = None
294 rangeIpp = None
295 rangeIpp = None #variables innecesarias?
295 rangeTxA = None
296 rangeTxA = None
296 rangeTxB = None
297 rangeTxB = None
297 structure = RADAR_STRUCTURE
298 structure = RADAR_STRUCTURE
298 __size = None
299 __size = None
300 ################################################
301 ippSeconds = None
302 frequency = None
303 sampleRate = None
304 nOsamp = None
305 channelList = []
306 azimuthList = []
307 elevationList =[]
308 codeList = []
309 nChannels = 1
310 heightList = []
311 heightResolution = None
312
299
313
300 def __init__(self, expType=2, nTx=1,
314 def __init__(self, expType=2, nTx=1,
301 ipp=None, txA=0, txB=0,
315 ipp=None, txA=0, txB=0,
302 nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None,
316 nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None,
303 numTaus=0, line6Function=0, line5Function=0, fClock=None,
317 numTaus=0, line6Function=0, line5Function=0, fClock=None,
304 prePulseBefore=0, prePulseAfter=0,
318 prePulseBefore=0, prePulseAfter=0,
305 codeType=0, nCode=0, nBaud=0, code=[],
319 codeType=0, nCode=0, nBaud=0, code=[],nOsamp = None, frequency = None,sampleRate=None,
306 flip1=0, flip2=0):
320 flip1=0, flip2=0, nChannels=1):
307
321
308 # self.size = 116
322 # self.size = 116
309 self.expType = expType
323 self.expType = expType
310 self.nTx = nTx
324 self.nTx = nTx
311 self.ipp = ipp
325 self.ipp = ipp
312 self.txA = txA
326 self.txA = txA
313 self.txB = txB
327 self.txB = txB
314 self.rangeIpp = ipp
328 self.rangeIpp = ipp
315 self.rangeTxA = txA
329 self.rangeTxA = txA
316 self.rangeTxB = txB
330 self.rangeTxB = txB
317
331 self.frequency = frequency
332 self.sampleRate = sampleRate
318 self.nWindows = nWindows
333 self.nWindows = nWindows
319 self.numTaus = numTaus
334 self.numTaus = numTaus
320 self.codeType = codeType
335 self.codeType = codeType
321 self.line6Function = line6Function
336 self.line6Function = line6Function
322 self.line5Function = line5Function
337 self.line5Function = line5Function
323 self.fClock = fClock
338 self.fClock = fClock
324 self.prePulseBefore = prePulseBefore
339 self.prePulseBefore = prePulseBefore
325 self.prePulseAfter = prePulseAfter
340 self.prePulseAfter = prePulseAfter
326
341
327 self.nHeights = nHeights
342 self.nHeights = nHeights
328 self.firstHeight = firstHeight
343 self.firstHeight = firstHeight
329 self.deltaHeight = deltaHeight
344 self.deltaHeight = deltaHeight
330 self.samplesWin = nHeights
345 self.samplesWin = nHeights
331
346
332 self.nCode = nCode
347 self.nCode = nCode
333 self.nBaud = nBaud
348 self.nBaud = nBaud
334 self.code = code
349 self.code = code
350 self.nOsamp = nOsamp
335 self.flip1 = flip1
351 self.flip1 = flip1
336 self.flip2 = flip2
352 self.flip2 = flip2
337
353
338 self.code_size = int(numpy.ceil(self.nBaud / 32.)) * self.nCode * 4
354 self.code_size = int(numpy.ceil(self.nBaud / 32.)) * self.nCode * 4
339 # self.dynamic = numpy.array([],numpy.dtype('byte'))
355 # self.dynamic = numpy.array([],numpy.dtype('byte'))
340
356
341 if self.fClock is None and self.deltaHeight is not None:
357 if self.fClock is None and self.deltaHeight is not None:
342 self.fClock = 0.15 / (deltaHeight * 1e-6) # 0.15Km / (height * 1u)
358 self.fClock = 0.15 / (deltaHeight * 1e-6) # 0.15Km / (height * 1u)
343
359
344 def read(self, fp):
360 def read(self, fp):
345 self.length = 0
361 self.length = 0
346 try:
362 try:
347 startFp = fp.tell()
363 startFp = fp.tell()
348 except Exception as e:
364 except Exception as e:
349 startFp = None
365 startFp = None
350 pass
366 pass
351
367
352 try:
368 try:
353 if hasattr(fp, 'read'):
369 if hasattr(fp, 'read'):
354 header = numpy.fromfile(fp, RADAR_STRUCTURE, 1)
370 header = numpy.fromfile(fp, RADAR_STRUCTURE, 1)
355 else:
371 else:
356 header = numpy.fromstring(fp, RADAR_STRUCTURE, 1)
372 header = numpy.fromstring(fp, RADAR_STRUCTURE, 1)
357 self.length += header.nbytes
373 self.length += header.nbytes
358 except Exception as e:
374 except Exception as e:
359 print("RadarControllerHeader: " + str(e))
375 print("RadarControllerHeader: " + str(e))
360 return 0
376 return 0
361
377
362 size = int(header['nSize'][0])
378 size = int(header['nSize'][0])
363 self.expType = int(header['nExpType'][0])
379 self.expType = int(header['nExpType'][0])
364 self.nTx = int(header['nNTx'][0])
380 self.nTx = int(header['nNTx'][0])
365 self.ipp = float(header['fIpp'][0])
381 self.ipp = float(header['fIpp'][0])
366 self.txA = float(header['fTxA'][0])
382 self.txA = float(header['fTxA'][0])
367 self.txB = float(header['fTxB'][0])
383 self.txB = float(header['fTxB'][0])
368 self.nWindows = int(header['nNumWindows'][0])
384 self.nWindows = int(header['nNumWindows'][0])
369 self.numTaus = int(header['nNumTaus'][0])
385 self.numTaus = int(header['nNumTaus'][0])
370 self.codeType = int(header['nCodeType'][0])
386 self.codeType = int(header['nCodeType'][0])
371 self.line6Function = int(header['nLine6Function'][0])
387 self.line6Function = int(header['nLine6Function'][0])
372 self.line5Function = int(header['nLine5Function'][0])
388 self.line5Function = int(header['nLine5Function'][0])
373 self.fClock = float(header['fClock'][0])
389 self.fClock = float(header['fClock'][0])
374 self.prePulseBefore = int(header['nPrePulseBefore'][0])
390 self.prePulseBefore = int(header['nPrePulseBefore'][0])
375 self.prePulseAfter = int(header['nPrePulseAfter'][0])
391 self.prePulseAfter = int(header['nPrePulseAfter'][0])
376 self.rangeIpp = header['sRangeIPP'][0]
392 self.rangeIpp = header['sRangeIPP'][0]
377 self.rangeTxA = header['sRangeTxA'][0]
393 self.rangeTxA = header['sRangeTxA'][0]
378 self.rangeTxB = header['sRangeTxB'][0]
394 self.rangeTxB = header['sRangeTxB'][0]
379
395
380 try:
396 try:
381 if hasattr(fp, 'read'):
397 if hasattr(fp, 'read'):
382 samplingWindow = numpy.fromfile(
398 samplingWindow = numpy.fromfile(
383 fp, SAMPLING_STRUCTURE, self.nWindows)
399 fp, SAMPLING_STRUCTURE, self.nWindows)
384 else:
400 else:
385 samplingWindow = numpy.fromstring(
401 samplingWindow = numpy.fromstring(
386 fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
402 fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
387 self.length += samplingWindow.nbytes
403 self.length += samplingWindow.nbytes
388 except Exception as e:
404 except Exception as e:
389 print("RadarControllerHeader: " + str(e))
405 print("RadarControllerHeader: " + str(e))
390 return 0
406 return 0
391 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
407 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
392 self.firstHeight = samplingWindow['h0']
408 self.firstHeight = samplingWindow['h0']
393 self.deltaHeight = samplingWindow['dh']
409 self.deltaHeight = samplingWindow['dh']
394 self.samplesWin = samplingWindow['nsa']
410 self.samplesWin = samplingWindow['nsa']
395
411
396 try:
412 try:
397 if hasattr(fp, 'read'):
413 if hasattr(fp, 'read'):
398 self.Taus = numpy.fromfile(fp, '<f4', self.numTaus)
414 self.Taus = numpy.fromfile(fp, '<f4', self.numTaus)
399 else:
415 else:
400 self.Taus = numpy.fromstring(
416 self.Taus = numpy.fromstring(
401 fp[self.length:], '<f4', self.numTaus)
417 fp[self.length:], '<f4', self.numTaus)
402 self.length += self.Taus.nbytes
418 self.length += self.Taus.nbytes
403 except Exception as e:
419 except Exception as e:
404 print("RadarControllerHeader: " + str(e))
420 print("RadarControllerHeader: " + str(e))
405 return 0
421 return 0
406
422
407 self.code_size = 0
423 self.code_size = 0
408 if self.codeType != 0:
424 if self.codeType != 0:
409
425
410 try:
426 try:
411 if hasattr(fp, 'read'):
427 if hasattr(fp, 'read'):
412 self.nCode = numpy.fromfile(fp, '<u4', 1)[0]
428 self.nCode = numpy.fromfile(fp, '<u4', 1)[0]
413 self.length += self.nCode.nbytes
429 self.length += self.nCode.nbytes
414 self.nBaud = numpy.fromfile(fp, '<u4', 1)[0]
430 self.nBaud = numpy.fromfile(fp, '<u4', 1)[0]
415 self.length += self.nBaud.nbytes
431 self.length += self.nBaud.nbytes
416 else:
432 else:
417 self.nCode = numpy.fromstring(
433 self.nCode = numpy.fromstring(
418 fp[self.length:], '<u4', 1)[0]
434 fp[self.length:], '<u4', 1)[0]
419 self.length += self.nCode.nbytes
435 self.length += self.nCode.nbytes
420 self.nBaud = numpy.fromstring(
436 self.nBaud = numpy.fromstring(
421 fp[self.length:], '<u4', 1)[0]
437 fp[self.length:], '<u4', 1)[0]
422 self.length += self.nBaud.nbytes
438 self.length += self.nBaud.nbytes
423 except Exception as e:
439 except Exception as e:
424 print("RadarControllerHeader: " + str(e))
440 print("RadarControllerHeader: " + str(e))
425 return 0
441 return 0
426 code = numpy.empty([self.nCode, self.nBaud], dtype='i1')
442 code = numpy.empty([self.nCode, self.nBaud], dtype='i1')
427
443
428 for ic in range(self.nCode):
444 for ic in range(self.nCode):
429 try:
445 try:
430 if hasattr(fp, 'read'):
446 if hasattr(fp, 'read'):
431 temp = numpy.fromfile(fp, 'u4', int(
447 temp = numpy.fromfile(fp, 'u4', int(
432 numpy.ceil(self.nBaud / 32.)))
448 numpy.ceil(self.nBaud / 32.)))
433 else:
449 else:
434 temp = numpy.fromstring(
450 temp = numpy.fromstring(
435 fp, 'u4', int(numpy.ceil(self.nBaud / 32.)))
451 fp, 'u4', int(numpy.ceil(self.nBaud / 32.)))
436 self.length += temp.nbytes
452 self.length += temp.nbytes
437 except Exception as e:
453 except Exception as e:
438 print("RadarControllerHeader: " + str(e))
454 print("RadarControllerHeader: " + str(e))
439 return 0
455 return 0
440
456
441 for ib in range(self.nBaud - 1, -1, -1):
457 for ib in range(self.nBaud - 1, -1, -1):
442 code[ic, ib] = temp[int(ib / 32)] % 2
458 code[ic, ib] = temp[int(ib / 32)] % 2
443 temp[int(ib / 32)] = temp[int(ib / 32)] / 2
459 temp[int(ib / 32)] = temp[int(ib / 32)] / 2
444
460
445 self.code = 2.0 * code - 1.0
461 self.code = 2.0 * code - 1.0
446 self.code_size = int(numpy.ceil(self.nBaud / 32.)) * self.nCode * 4
462 self.code_size = int(numpy.ceil(self.nBaud / 32.)) * self.nCode * 4
447
463
448 # if self.line5Function == RCfunction.FLIP:
464 # if self.line5Function == RCfunction.FLIP:
449 # self.flip1 = numpy.fromfile(fp,'<u4',1)
465 # self.flip1 = numpy.fromfile(fp,'<u4',1)
450 #
466 #
451 # if self.line6Function == RCfunction.FLIP:
467 # if self.line6Function == RCfunction.FLIP:
452 # self.flip2 = numpy.fromfile(fp,'<u4',1)
468 # self.flip2 = numpy.fromfile(fp,'<u4',1)
453 if startFp is not None:
469 if startFp is not None:
454 endFp = size + startFp
470 endFp = size + startFp
455
471
456 if fp.tell() != endFp:
472 if fp.tell() != endFp:
457 # fp.seek(endFp)
473 # fp.seek(endFp)
458 print("%s: Radar Controller Header size is not consistent: from data [%d] != from header field [%d]" % (fp.name, fp.tell() - startFp, size))
474 print("%s: Radar Controller Header size is not consistent: from data [%d] != from header field [%d]" % (fp.name, fp.tell() - startFp, size))
459 # return 0
475 # return 0
460
476
461 if fp.tell() > endFp:
477 if fp.tell() > endFp:
462 sys.stderr.write(
478 sys.stderr.write(
463 "Warning %s: Size value read from Radar Controller header is lower than it has to be\n" % fp.name)
479 "Warning %s: Size value read from Radar Controller header is lower than it has to be\n" % fp.name)
464 # return 0
480 # return 0
465
481
466 if fp.tell() < endFp:
482 if fp.tell() < endFp:
467 sys.stderr.write(
483 sys.stderr.write(
468 "Warning %s: Size value read from Radar Controller header is greater than it has to be\n" % fp.name)
484 "Warning %s: Size value read from Radar Controller header is greater than it has to be\n" % fp.name)
469
485
470 return 1
486 return 1
471
487
472 def toString(self):
488 def toString(self):
473 #attrs = dir(self)
489 #attrs = dir(self)
474 s = ""
490 s = ""
475 for attribute, value in self.__dict__.items():
491 for attribute, value in self.__dict__.items():
476 if value!=None and value!= 0:
492 if value!=None and value!= 0:
477 s += '{:18s}'.format(str(attribute)) +'\t'+str(value)+'\n'
493 s += '{:18s}'.format(str(attribute)) +'\t'+str(value)+'\n'
478 #s += str(att) +'\t'+str(getattr(self, att))+'\n'
494 #s += str(att) +'\t'+str(getattr(self, att))+'\n'
479 return s
495 return s
480
496
481 def write(self, fp):
497 def write(self, fp):
482
498
483 headerTuple = (self.size,
499 headerTuple = (self.size,
484 self.expType,
500 self.expType,
485 self.nTx,
501 self.nTx,
486 self.ipp,
502 self.ipp,
487 self.txA,
503 self.txA,
488 self.txB,
504 self.txB,
489 self.nWindows,
505 self.nWindows,
490 self.numTaus,
506 self.numTaus,
491 self.codeType,
507 self.codeType,
492 self.line6Function,
508 self.line6Function,
493 self.line5Function,
509 self.line5Function,
494 self.fClock,
510 self.fClock,
495 self.prePulseBefore,
511 self.prePulseBefore,
496 self.prePulseAfter,
512 self.prePulseAfter,
497 self.rangeIpp,
513 self.rangeIpp,
498 self.rangeTxA,
514 self.rangeTxA,
499 self.rangeTxB)
515 self.rangeTxB)
500
516
501 header = numpy.array(headerTuple, RADAR_STRUCTURE)
517 header = numpy.array(headerTuple, RADAR_STRUCTURE)
502 header.tofile(fp)
518 header.tofile(fp)
503
519
504 sampleWindowTuple = (
520 sampleWindowTuple = (
505 self.firstHeight, self.deltaHeight, self.samplesWin)
521 self.firstHeight, self.deltaHeight, self.samplesWin)
506 samplingWindow = numpy.array(sampleWindowTuple, SAMPLING_STRUCTURE)
522 samplingWindow = numpy.array(sampleWindowTuple, SAMPLING_STRUCTURE)
507 samplingWindow.tofile(fp)
523 samplingWindow.tofile(fp)
508
524
509 if self.numTaus > 0:
525 if self.numTaus > 0:
510 self.Taus.tofile(fp)
526 self.Taus.tofile(fp)
511
527
512 if self.codeType != 0:
528 if self.codeType != 0:
513 nCode = numpy.array(self.nCode, '<u4')
529 nCode = numpy.array(self.nCode, '<u4')
514 nCode.tofile(fp)
530 nCode.tofile(fp)
515 nBaud = numpy.array(self.nBaud, '<u4')
531 nBaud = numpy.array(self.nBaud, '<u4')
516 nBaud.tofile(fp)
532 nBaud.tofile(fp)
517 code1 = (self.code + 1.0) / 2.
533 code1 = (self.code + 1.0) / 2.
518
534
519 for ic in range(self.nCode):
535 for ic in range(self.nCode):
520 tempx = numpy.zeros(int(numpy.ceil(self.nBaud / 32.)))
536 tempx = numpy.zeros(int(numpy.ceil(self.nBaud / 32.)))
521 start = 0
537 start = 0
522 end = 32
538 end = 32
523 for i in range(len(tempx)):
539 for i in range(len(tempx)):
524 code_selected = code1[ic, start:end]
540 code_selected = code1[ic, start:end]
525 for j in range(len(code_selected) - 1, -1, -1):
541 for j in range(len(code_selected) - 1, -1, -1):
526 if code_selected[j] == 1:
542 if code_selected[j] == 1:
527 tempx[i] = tempx[i] + \
543 tempx[i] = tempx[i] + \
528 2**(len(code_selected) - 1 - j)
544 2**(len(code_selected) - 1 - j)
529 start = start + 32
545 start = start + 32
530 end = end + 32
546 end = end + 32
531
547
532 tempx = tempx.astype('u4')
548 tempx = tempx.astype('u4')
533 tempx.tofile(fp)
549 tempx.tofile(fp)
534
550
535 # if self.line5Function == RCfunction.FLIP:
551 # if self.line5Function == RCfunction.FLIP:
536 # self.flip1.tofile(fp)
552 # self.flip1.tofile(fp)
537 #
553 #
538 # if self.line6Function == RCfunction.FLIP:
554 # if self.line6Function == RCfunction.FLIP:
539 # self.flip2.tofile(fp)
555 # self.flip2.tofile(fp)
540
556
541 return 1
557 return 1
542
558
543 def get_ippSeconds(self):
559 def get_ippSeconds(self):
544 '''
560 '''
545 '''
561 '''
546 ippSeconds = 2.0 * 1000 * self.ipp / SPEED_OF_LIGHT
562 ippSeconds = 2.0 * 1000 * self.ipp / SPEED_OF_LIGHT
547
563
548 return ippSeconds
564 return ippSeconds
549
565
550 def set_ippSeconds(self, ippSeconds):
566 def set_ippSeconds(self, ippSeconds):
551 '''
567 '''
552 '''
568 '''
553
569
554 self.ipp = ippSeconds * SPEED_OF_LIGHT / (2.0 * 1000)
570 self.ipp = ippSeconds * SPEED_OF_LIGHT / (2.0 * 1000)
555
571
556 return
572 return
557
573
558 def get_size(self):
574 def get_size(self):
559
575
560 self.__size = 116 + 12 * self.nWindows + 4 * self.numTaus
576 self.__size = 116 + 12 * self.nWindows + 4 * self.numTaus
561
577
562 if self.codeType != 0:
578 if self.codeType != 0:
563 self.__size += 4 + 4 + 4 * self.nCode * \
579 self.__size += 4 + 4 + 4 * self.nCode * \
564 numpy.ceil(self.nBaud / 32.)
580 numpy.ceil(self.nBaud / 32.)
565
581
566 return self.__size
582 return self.__size
567
583
568 def set_size(self, value):
584 def set_size(self, value):
569
585
570 raise IOError("size is a property and it cannot be set, just read")
586 raise IOError("size is a property and it cannot be set, just read")
571
587
572 return
588 return
573
589
574 ippSeconds = property(get_ippSeconds, set_ippSeconds)
590 ippSeconds = property(get_ippSeconds, set_ippSeconds)
575 size = property(get_size, set_size)
591 size = property(get_size, set_size)
576
592
577
593
578 class ProcessingHeader(Header):
594 class ProcessingHeader(Header):
579
595
580 # size = None
596 # size = None
597
581 dtype = None
598 dtype = None
582 blockSize = None
599 blockSize = None
583 profilesPerBlock = None
600 profilesPerBlock = None
584 dataBlocksPerFile = None
601 dataBlocksPerFile = None
585 nWindows = None
602 nWindows = None
586 processFlags = None
603 processFlags = None
587 nCohInt = None
604 nCohInt = None
588 nIncohInt = None
605 nIncohInt = None
589 totalSpectra = None
606 totalSpectra = None
590 structure = PROCESSING_STRUCTURE
607 structure = PROCESSING_STRUCTURE
591 flag_dc = None
608 flag_dc = None
592 flag_cspc = None
609 flag_cspc = None
610 #########################################################
611 nFFTPoints = None
612 nSamplesFFT = None
613 channelList = []
614 azimuthList = []
615 elevationList =[]
616 codeList = []
617 nChannels = 1
618 heightList = []
619 ipp = None
620 ippSeconds = None
621 timeIncohInt = None
622 #################
623 rangeIpp = None
624 heightResolution = None
593
625
594 def __init__(self, dtype=0, blockSize=0, profilesPerBlock=0, dataBlocksPerFile=0, nWindows=0, processFlags=0, nCohInt=0,
626 def __init__(self, dtype=0, blockSize=0, profilesPerBlock=0, dataBlocksPerFile=0, nWindows=0, processFlags=0, nCohInt=0,
595 nIncohInt=0, totalSpectra=0, nHeights=0, firstHeight=0, deltaHeight=0, samplesWin=0, spectraComb=0, nCode=0,
627 nIncohInt=0, totalSpectra=0, nHeights=0, firstHeight=0, deltaHeight=0, samplesWin=0, spectraComb=0, nCode=0,
596 code=0, nBaud=None, shif_fft=False, flag_dc=False, flag_cspc=False, flag_decode=False, flag_deflip=False
628 code=0, nBaud=None, shif_fft=False, flag_dc=False, flag_cspc=False, flag_decode=False, flag_deflip=False
597 ):
629 ):
598
630
599 # self.size = 0
631 # self.size = 0
600 self.dtype = dtype
632 self.dtype = dtype
601 self.blockSize = blockSize
633 self.blockSize = blockSize
602 self.profilesPerBlock = 0
634 self.profilesPerBlock = 0
603 self.dataBlocksPerFile = 0
635 self.dataBlocksPerFile = 0
604 self.nWindows = 0
636 self.nWindows = 0
605 self.processFlags = 0
637 self.processFlags = 0
606 self.nCohInt = 0
638 self.nCohInt = 0
607 self.nIncohInt = 0
639 self.nIncohInt = 0
608 self.totalSpectra = 0
640 self.totalSpectra = 0
609
641
610 self.nHeights = 0
642 self.nHeights = 0
611 self.firstHeight = 0
643 self.firstHeight = 0
612 self.deltaHeight = 0
644 self.deltaHeight = 0
613 self.samplesWin = 0
645 self.samplesWin = 0
614 self.spectraComb = 0
646 self.spectraComb = 0
615 self.nCode = None
647 self.nCode = None
616 self.code = None
648 self.code = None
617 self.nBaud = None
649 self.nBaud = None
618
650
619 self.shif_fft = False
651 self.shif_fft = False
620 self.flag_dc = False
652 self.flag_dc = False
621 self.flag_cspc = False
653 self.flag_cspc = False
622 self.flag_decode = False
654 self.flag_decode = False
623 self.flag_deflip = False
655 self.flag_deflip = False
624 self.length = 0
656 self.length = 0
625
657
658
626 def read(self, fp):
659 def read(self, fp):
627 self.length = 0
660 self.length = 0
628 try:
661 try:
629 startFp = fp.tell()
662 startFp = fp.tell()
630 except Exception as e:
663 except Exception as e:
631 startFp = None
664 startFp = None
632 pass
665 pass
633
666
634 try:
667 try:
635 if hasattr(fp, 'read'):
668 if hasattr(fp, 'read'):
636 header = numpy.fromfile(fp, PROCESSING_STRUCTURE, 1)
669 header = numpy.fromfile(fp, PROCESSING_STRUCTURE, 1)
637 else:
670 else:
638 header = numpy.fromstring(fp, PROCESSING_STRUCTURE, 1)
671 header = numpy.fromstring(fp, PROCESSING_STRUCTURE, 1)
639 self.length += header.nbytes
672 self.length += header.nbytes
640 except Exception as e:
673 except Exception as e:
641 print("ProcessingHeader: " + str(e))
674 print("ProcessingHeader: " + str(e))
642 return 0
675 return 0
643
676
644 size = int(header['nSize'][0])
677 size = int(header['nSize'][0])
645 self.dtype = int(header['nDataType'][0])
678 self.dtype = int(header['nDataType'][0])
646 self.blockSize = int(header['nSizeOfDataBlock'][0])
679 self.blockSize = int(header['nSizeOfDataBlock'][0])
647 self.profilesPerBlock = int(header['nProfilesperBlock'][0])
680 self.profilesPerBlock = int(header['nProfilesperBlock'][0])
648 self.dataBlocksPerFile = int(header['nDataBlocksperFile'][0])
681 self.dataBlocksPerFile = int(header['nDataBlocksperFile'][0])
649 self.nWindows = int(header['nNumWindows'][0])
682 self.nWindows = int(header['nNumWindows'][0])
650 self.processFlags = header['nProcessFlags']
683 self.processFlags = header['nProcessFlags']
651 self.nCohInt = int(header['nCoherentIntegrations'][0])
684 self.nCohInt = int(header['nCoherentIntegrations'][0])
652 self.nIncohInt = int(header['nIncoherentIntegrations'][0])
685 self.nIncohInt = int(header['nIncoherentIntegrations'][0])
653 self.totalSpectra = int(header['nTotalSpectra'][0])
686 self.totalSpectra = int(header['nTotalSpectra'][0])
654
687
655 try:
688 try:
656 if hasattr(fp, 'read'):
689 if hasattr(fp, 'read'):
657 samplingWindow = numpy.fromfile(
690 samplingWindow = numpy.fromfile(
658 fp, SAMPLING_STRUCTURE, self.nWindows)
691 fp, SAMPLING_STRUCTURE, self.nWindows)
659 else:
692 else:
660 samplingWindow = numpy.fromstring(
693 samplingWindow = numpy.fromstring(
661 fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
694 fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
662 self.length += samplingWindow.nbytes
695 self.length += samplingWindow.nbytes
663 except Exception as e:
696 except Exception as e:
664 print("ProcessingHeader: " + str(e))
697 print("ProcessingHeader: " + str(e))
665 return 0
698 return 0
666
699
667 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
700 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
668 self.firstHeight = float(samplingWindow['h0'][0])
701 self.firstHeight = float(samplingWindow['h0'][0])
669 self.deltaHeight = float(samplingWindow['dh'][0])
702 self.deltaHeight = float(samplingWindow['dh'][0])
670 self.samplesWin = samplingWindow['nsa'][0]
703 self.samplesWin = samplingWindow['nsa'][0]
671
704
672 try:
705 try:
673 if hasattr(fp, 'read'):
706 if hasattr(fp, 'read'):
674 self.spectraComb = numpy.fromfile(
707 self.spectraComb = numpy.fromfile(
675 fp, 'u1', 2 * self.totalSpectra)
708 fp, 'u1', 2 * self.totalSpectra)
676 else:
709 else:
677 self.spectraComb = numpy.fromstring(
710 self.spectraComb = numpy.fromstring(
678 fp[self.length:], 'u1', 2 * self.totalSpectra)
711 fp[self.length:], 'u1', 2 * self.totalSpectra)
679 self.length += self.spectraComb.nbytes
712 self.length += self.spectraComb.nbytes
680 except Exception as e:
713 except Exception as e:
681 print("ProcessingHeader: " + str(e))
714 print("ProcessingHeader: " + str(e))
682 return 0
715 return 0
683
716
684 if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
717 if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
685 self.nCode = int(numpy.fromfile(fp, '<u4', 1))
718 self.nCode = int(numpy.fromfile(fp, '<u4', 1))
686 self.nBaud = int(numpy.fromfile(fp, '<u4', 1))
719 self.nBaud = int(numpy.fromfile(fp, '<u4', 1))
687 self.code = numpy.fromfile(
720 self.code = numpy.fromfile(
688 fp, '<f4', self.nCode * self.nBaud).reshape(self.nCode, self.nBaud)
721 fp, '<f4', self.nCode * self.nBaud).reshape(self.nCode, self.nBaud)
689
722
690 if ((self.processFlags & PROCFLAG.EXP_NAME_ESP) == PROCFLAG.EXP_NAME_ESP):
723 if ((self.processFlags & PROCFLAG.EXP_NAME_ESP) == PROCFLAG.EXP_NAME_ESP):
691 exp_name_len = int(numpy.fromfile(fp, '<u4', 1))
724 exp_name_len = int(numpy.fromfile(fp, '<u4', 1))
692 exp_name = numpy.fromfile(fp, 'u1', exp_name_len + 1)
725 exp_name = numpy.fromfile(fp, 'u1', exp_name_len + 1)
693
726
694 if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA):
727 if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA):
695 self.shif_fft = True
728 self.shif_fft = True
696 else:
729 else:
697 self.shif_fft = False
730 self.shif_fft = False
698
731
699 if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC):
732 if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC):
700 self.flag_dc = True
733 self.flag_dc = True
701 else:
734 else:
702 self.flag_dc = False
735 self.flag_dc = False
703
736
704 if ((self.processFlags & PROCFLAG.DECODE_DATA) == PROCFLAG.DECODE_DATA):
737 if ((self.processFlags & PROCFLAG.DECODE_DATA) == PROCFLAG.DECODE_DATA):
705 self.flag_decode = True
738 self.flag_decode = True
706 else:
739 else:
707 self.flag_decode = False
740 self.flag_decode = False
708
741
709 if ((self.processFlags & PROCFLAG.DEFLIP_DATA) == PROCFLAG.DEFLIP_DATA):
742 if ((self.processFlags & PROCFLAG.DEFLIP_DATA) == PROCFLAG.DEFLIP_DATA):
710 self.flag_deflip = True
743 self.flag_deflip = True
711 else:
744 else:
712 self.flag_deflip = False
745 self.flag_deflip = False
713
746
714 nChannels = 0
747 nChannels = 0
715 nPairs = 0
748 nPairs = 0
716 pairList = []
749 pairList = []
717
750
718 for i in range(0, self.totalSpectra * 2, 2):
751 for i in range(0, self.totalSpectra * 2, 2):
719 if self.spectraComb[i] == self.spectraComb[i + 1]:
752 if self.spectraComb[i] == self.spectraComb[i + 1]:
720 nChannels = nChannels + 1 # par de canales iguales
753 nChannels = nChannels + 1 # par de canales iguales
721 else:
754 else:
722 nPairs = nPairs + 1 # par de canales diferentes
755 nPairs = nPairs + 1 # par de canales diferentes
723 pairList.append((self.spectraComb[i], self.spectraComb[i + 1]))
756 pairList.append((self.spectraComb[i], self.spectraComb[i + 1]))
724
757
725 self.flag_cspc = False
758 self.flag_cspc = False
726 if nPairs > 0:
759 if nPairs > 0:
727 self.flag_cspc = True
760 self.flag_cspc = True
728
761
729 if startFp is not None:
762 if startFp is not None:
730 endFp = size + startFp
763 endFp = size + startFp
731 if fp.tell() > endFp:
764 if fp.tell() > endFp:
732 sys.stderr.write(
765 sys.stderr.write(
733 "Warning: Processing header size is lower than it has to be")
766 "Warning: Processing header size is lower than it has to be")
734 return 0
767 return 0
735
768
736 if fp.tell() < endFp:
769 if fp.tell() < endFp:
737 sys.stderr.write(
770 sys.stderr.write(
738 "Warning: Processing header size is greater than it is considered")
771 "Warning: Processing header size is greater than it is considered")
739
772
740 return 1
773 return 1
741
774
742 def write(self, fp):
775 def write(self, fp):
743 # Clear DEFINE_PROCESS_CODE
776 # Clear DEFINE_PROCESS_CODE
744 self.processFlags = self.processFlags & (~PROCFLAG.DEFINE_PROCESS_CODE)
777 self.processFlags = self.processFlags & (~PROCFLAG.DEFINE_PROCESS_CODE)
745
778
746 headerTuple = (self.size,
779 headerTuple = (self.size,
747 self.dtype,
780 self.dtype,
748 self.blockSize,
781 self.blockSize,
749 self.profilesPerBlock,
782 self.profilesPerBlock,
750 self.dataBlocksPerFile,
783 self.dataBlocksPerFile,
751 self.nWindows,
784 self.nWindows,
752 self.processFlags,
785 self.processFlags,
753 self.nCohInt,
786 self.nCohInt,
754 self.nIncohInt,
787 self.nIncohInt,
755 self.totalSpectra)
788 self.totalSpectra)
756
789
757 header = numpy.array(headerTuple, PROCESSING_STRUCTURE)
790 header = numpy.array(headerTuple, PROCESSING_STRUCTURE)
758 header.tofile(fp)
791 header.tofile(fp)
759
792
760 if self.nWindows != 0:
793 if self.nWindows != 0:
761 sampleWindowTuple = (
794 sampleWindowTuple = (
762 self.firstHeight, self.deltaHeight, self.samplesWin)
795 self.firstHeight, self.deltaHeight, self.samplesWin)
763 samplingWindow = numpy.array(sampleWindowTuple, SAMPLING_STRUCTURE)
796 samplingWindow = numpy.array(sampleWindowTuple, SAMPLING_STRUCTURE)
764 samplingWindow.tofile(fp)
797 samplingWindow.tofile(fp)
765
798
766 if self.totalSpectra != 0:
799 if self.totalSpectra != 0:
767 # spectraComb = numpy.array([],numpy.dtype('u1'))
800 # spectraComb = numpy.array([],numpy.dtype('u1'))
768 spectraComb = self.spectraComb
801 spectraComb = self.spectraComb
769 spectraComb.tofile(fp)
802 spectraComb.tofile(fp)
770
803
771 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
804 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
772 # nCode = numpy.array([self.nCode], numpy.dtype('u4')) #Probar con un dato que almacene codigo, hasta el momento no se hizo la prueba
805 # nCode = numpy.array([self.nCode], numpy.dtype('u4')) #Probar con un dato que almacene codigo, hasta el momento no se hizo la prueba
773 # nCode.tofile(fp)
806 # nCode.tofile(fp)
774 #
807 #
775 # nBaud = numpy.array([self.nBaud], numpy.dtype('u4'))
808 # nBaud = numpy.array([self.nBaud], numpy.dtype('u4'))
776 # nBaud.tofile(fp)
809 # nBaud.tofile(fp)
777 #
810 #
778 # code = self.code.reshape(self.nCode*self.nBaud)
811 # code = self.code.reshape(self.nCode*self.nBaud)
779 # code = code.astype(numpy.dtype('<f4'))
812 # code = code.astype(numpy.dtype('<f4'))
780 # code.tofile(fp)
813 # code.tofile(fp)
781
814
782 return 1
815 return 1
783
816
784 def get_size(self):
817 def get_size(self):
785
818
786 self.__size = 40 + 12 * self.nWindows + 2 * self.totalSpectra
819 self.__size = 40 + 12 * self.nWindows + 2 * self.totalSpectra
787
820
788 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
821 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
789 # self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
822 # self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
790 # self.__size += 4 + 4 + 4 * self.nCode * self.nBaud
823 # self.__size += 4 + 4 + 4 * self.nCode * self.nBaud
791
824
792 return self.__size
825 return self.__size
793
826
794 def set_size(self, value):
827 def set_size(self, value):
795
828
796 raise IOError("size is a property and it cannot be set, just read")
829 raise IOError("size is a property and it cannot be set, just read")
797
830
798 return
831 return
799
832
800 size = property(get_size, set_size)
833 size = property(get_size, set_size)
801
834
802
835
803 class RCfunction:
836 class RCfunction:
804 NONE = 0
837 NONE = 0
805 FLIP = 1
838 FLIP = 1
806 CODE = 2
839 CODE = 2
807 SAMPLING = 3
840 SAMPLING = 3
808 LIN6DIV256 = 4
841 LIN6DIV256 = 4
809 SYNCHRO = 5
842 SYNCHRO = 5
810
843
811
844
812 class nCodeType:
845 class nCodeType:
813 NONE = 0
846 NONE = 0
814 USERDEFINE = 1
847 USERDEFINE = 1
815 BARKER2 = 2
848 BARKER2 = 2
816 BARKER3 = 3
849 BARKER3 = 3
817 BARKER4 = 4
850 BARKER4 = 4
818 BARKER5 = 5
851 BARKER5 = 5
819 BARKER7 = 6
852 BARKER7 = 6
820 BARKER11 = 7
853 BARKER11 = 7
821 BARKER13 = 8
854 BARKER13 = 8
822 AC128 = 9
855 AC128 = 9
823 COMPLEMENTARYCODE2 = 10
856 COMPLEMENTARYCODE2 = 10
824 COMPLEMENTARYCODE4 = 11
857 COMPLEMENTARYCODE4 = 11
825 COMPLEMENTARYCODE8 = 12
858 COMPLEMENTARYCODE8 = 12
826 COMPLEMENTARYCODE16 = 13
859 COMPLEMENTARYCODE16 = 13
827 COMPLEMENTARYCODE32 = 14
860 COMPLEMENTARYCODE32 = 14
828 COMPLEMENTARYCODE64 = 15
861 COMPLEMENTARYCODE64 = 15
829 COMPLEMENTARYCODE128 = 16
862 COMPLEMENTARYCODE128 = 16
830 CODE_BINARY28 = 17
863 CODE_BINARY28 = 17
831
864
832
865
833 class PROCFLAG:
866 class PROCFLAG:
834
867
835 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
868 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
836 DECODE_DATA = numpy.uint32(0x00000002)
869 DECODE_DATA = numpy.uint32(0x00000002)
837 SPECTRA_CALC = numpy.uint32(0x00000004)
870 SPECTRA_CALC = numpy.uint32(0x00000004)
838 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
871 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
839 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
872 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
840 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
873 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
841
874
842 DATATYPE_CHAR = numpy.uint32(0x00000040)
875 DATATYPE_CHAR = numpy.uint32(0x00000040)
843 DATATYPE_SHORT = numpy.uint32(0x00000080)
876 DATATYPE_SHORT = numpy.uint32(0x00000080)
844 DATATYPE_LONG = numpy.uint32(0x00000100)
877 DATATYPE_LONG = numpy.uint32(0x00000100)
845 DATATYPE_INT64 = numpy.uint32(0x00000200)
878 DATATYPE_INT64 = numpy.uint32(0x00000200)
846 DATATYPE_FLOAT = numpy.uint32(0x00000400)
879 DATATYPE_FLOAT = numpy.uint32(0x00000400)
847 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
880 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
848
881
849 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
882 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
850 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
883 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
851 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
884 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
852
885
853 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
886 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
854 DEFLIP_DATA = numpy.uint32(0x00010000)
887 DEFLIP_DATA = numpy.uint32(0x00010000)
855 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
888 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
856
889
857 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
890 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
858 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
891 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
859 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
892 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
860 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
893 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
861 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
894 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
862
895
863 EXP_NAME_ESP = numpy.uint32(0x00200000)
896 EXP_NAME_ESP = numpy.uint32(0x00200000)
864 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
897 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
865
898
866 OPERATION_MASK = numpy.uint32(0x0000003F)
899 OPERATION_MASK = numpy.uint32(0x0000003F)
867 DATATYPE_MASK = numpy.uint32(0x00000FC0)
900 DATATYPE_MASK = numpy.uint32(0x00000FC0)
868 DATAARRANGE_MASK = numpy.uint32(0x00007000)
901 DATAARRANGE_MASK = numpy.uint32(0x00007000)
869 ACQ_SYS_MASK = numpy.uint32(0x001C0000)
902 ACQ_SYS_MASK = numpy.uint32(0x001C0000)
870
903
871
904
872 dtype0 = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
905 dtype0 = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
873 dtype1 = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
906 dtype1 = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
874 dtype2 = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
907 dtype2 = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
875 dtype3 = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
908 dtype3 = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
876 dtype4 = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
909 dtype4 = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
877 dtype5 = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
910 dtype5 = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
878
911
879 NUMPY_DTYPE_LIST = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
912 NUMPY_DTYPE_LIST = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
880
913
881 PROCFLAG_DTYPE_LIST = [PROCFLAG.DATATYPE_CHAR,
914 PROCFLAG_DTYPE_LIST = [PROCFLAG.DATATYPE_CHAR,
882 PROCFLAG.DATATYPE_SHORT,
915 PROCFLAG.DATATYPE_SHORT,
883 PROCFLAG.DATATYPE_LONG,
916 PROCFLAG.DATATYPE_LONG,
884 PROCFLAG.DATATYPE_INT64,
917 PROCFLAG.DATATYPE_INT64,
885 PROCFLAG.DATATYPE_FLOAT,
918 PROCFLAG.DATATYPE_FLOAT,
886 PROCFLAG.DATATYPE_DOUBLE]
919 PROCFLAG.DATATYPE_DOUBLE]
887
920
888 DTYPE_WIDTH = [1, 2, 4, 8, 4, 8]
921 DTYPE_WIDTH = [1, 2, 4, 8, 4, 8]
889
922
890
923
891 def get_dtype_index(numpy_dtype):
924 def get_dtype_index(numpy_dtype):
892
925
893 index = None
926 index = None
894
927
895 for i in range(len(NUMPY_DTYPE_LIST)):
928 for i in range(len(NUMPY_DTYPE_LIST)):
896 if numpy_dtype == NUMPY_DTYPE_LIST[i]:
929 if numpy_dtype == NUMPY_DTYPE_LIST[i]:
897 index = i
930 index = i
898 break
931 break
899
932
900 return index
933 return index
901
934
902
935
903 def get_numpy_dtype(index):
936 def get_numpy_dtype(index):
904
937
905 return NUMPY_DTYPE_LIST[index]
938 return NUMPY_DTYPE_LIST[index]
906
939
907
940
908 def get_procflag_dtype(index):
941 def get_procflag_dtype(index):
909
942
910 return PROCFLAG_DTYPE_LIST[index]
943 return PROCFLAG_DTYPE_LIST[index]
911
944
912
945
913 def get_dtype_width(index):
946 def get_dtype_width(index):
914
947
915 return DTYPE_WIDTH[index]
948 return DTYPE_WIDTH[index]
@@ -1,1234 +1,1241
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """Classes to plot Spectra data
5 """Classes to plot Spectra data
6
6
7 """
7 """
8
8
9 import os
9 import os
10 import numpy
10 import numpy
11
11
12 from schainpy.model.graphics.jroplot_base import Plot, plt, log
12 from schainpy.model.graphics.jroplot_base import Plot, plt, log
13 from itertools import combinations
13 from itertools import combinations
14 from matplotlib.ticker import LinearLocator
14 from matplotlib.ticker import LinearLocator
15
15
16 class SpectraPlot(Plot):
16 class SpectraPlot(Plot):
17 '''
17 '''
18 Plot for Spectra data
18 Plot for Spectra data
19 '''
19 '''
20
20
21 CODE = 'spc'
21 CODE = 'spc'
22 colormap = 'jet'
22 colormap = 'jet'
23 plot_type = 'pcolor'
23 plot_type = 'pcolor'
24 buffering = False
24 buffering = False
25 channelList = []
25 channelList = []
26 elevationList = []
26 elevationList = []
27 azimuthList = []
27 azimuthList = []
28
28
29 def setup(self):
29 def setup(self):
30
30
31 self.nplots = len(self.data.channels)
31 self.nplots = len(self.data.channels)
32 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
32 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
33 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
33 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
34 self.height = 3.4 * self.nrows
34 self.height = 3.4 * self.nrows
35
35
36 self.cb_label = 'dB'
36 self.cb_label = 'dB'
37 if self.showprofile:
37 if self.showprofile:
38 self.width = 5.2 * self.ncols
38 self.width = 5.2 * self.ncols
39 else:
39 else:
40 self.width = 4.2* self.ncols
40 self.width = 4.2* self.ncols
41 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.12})
41 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.12})
42 self.ylabel = 'Range [km]'
42 self.ylabel = 'Range [km]'
43
43
44
44
45 def update_list(self,dataOut):
45 def update_list(self,dataOut):
46 if len(self.channelList) == 0:
46 if len(self.channelList) == 0:
47 self.channelList = dataOut.channelList
47 self.channelList = dataOut.channelList
48 if len(self.elevationList) == 0:
48 if len(self.elevationList) == 0:
49 self.elevationList = dataOut.elevationList
49 self.elevationList = dataOut.elevationList
50 if len(self.azimuthList) == 0:
50 if len(self.azimuthList) == 0:
51 self.azimuthList = dataOut.azimuthList
51 self.azimuthList = dataOut.azimuthList
52
52
53 def update(self, dataOut):
53 def update(self, dataOut):
54
54
55 self.update_list(dataOut)
55 self.update_list(dataOut)
56 data = {}
56 data = {}
57 meta = {}
57 meta = {}
58
58
59 #data['rti'] = dataOut.getPower()
59 #data['rti'] = dataOut.getPower()
60 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
60 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
61 noise = 10*numpy.log10(dataOut.getNoise()/norm)
61 noise = 10*numpy.log10(dataOut.getNoise()/norm)
62
62
63
63
64 z = numpy.zeros((dataOut.nChannels, dataOut.nFFTPoints, dataOut.nHeights))
64 z = numpy.zeros((dataOut.nChannels, dataOut.nFFTPoints, dataOut.nHeights))
65 for ch in range(dataOut.nChannels):
65 for ch in range(dataOut.nChannels):
66 if hasattr(dataOut.normFactor,'ndim'):
66 if hasattr(dataOut.normFactor,'ndim'):
67 if dataOut.normFactor.ndim > 1:
67 if dataOut.normFactor.ndim > 1:
68 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor[ch]))
68 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor[ch]))
69
69
70 else:
70 else:
71 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
71 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
72 else:
72 else:
73 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
73 z[ch] = (numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
74
74
75
75
76 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
76 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
77 spc = 10*numpy.log10(z)
77 spc = 10*numpy.log10(z)
78
78
79 data['spc'] = spc
79 data['spc'] = spc
80 print(spc[0].shape)
80 #print(spc[0].shape)
81 data['rti'] = spc.mean(axis=1)
81 data['rti'] = spc.mean(axis=1)
82 data['noise'] = noise
82 data['noise'] = noise
83 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
83 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
84 if self.CODE == 'spc_moments':
84 if self.CODE == 'spc_moments':
85 data['moments'] = dataOut.moments
85 data['moments'] = dataOut.moments
86
86
87 return data, meta
87 return data, meta
88
88
89 def plot(self):
89 def plot(self):
90 if self.xaxis == "frequency":
90 if self.xaxis == "frequency":
91 x = self.data.xrange[0]
91 x = self.data.xrange[0]
92 self.xlabel = "Frequency (kHz)"
92 self.xlabel = "Frequency (kHz)"
93 elif self.xaxis == "time":
93 elif self.xaxis == "time":
94 x = self.data.xrange[1]
94 x = self.data.xrange[1]
95 self.xlabel = "Time (ms)"
95 self.xlabel = "Time (ms)"
96 else:
96 else:
97 x = self.data.xrange[2]
97 x = self.data.xrange[2]
98 self.xlabel = "Velocity (m/s)"
98 self.xlabel = "Velocity (m/s)"
99
99
100 if self.CODE == 'spc_moments':
100 if self.CODE == 'spc_moments':
101 x = self.data.xrange[2]
101 x = self.data.xrange[2]
102 self.xlabel = "Velocity (m/s)"
102 self.xlabel = "Velocity (m/s)"
103
103
104 self.titles = []
104 self.titles = []
105 y = self.data.yrange
105 y = self.data.yrange
106 self.y = y
106 self.y = y
107
107
108 data = self.data[-1]
108 data = self.data[-1]
109 z = data['spc']
109 z = data['spc']
110 #print(z.shape, x.shape, y.shape)
110 #print(z.shape, x.shape, y.shape)
111 for n, ax in enumerate(self.axes):
111 for n, ax in enumerate(self.axes):
112 noise = self.data['noise'][n][0]
112 noise = self.data['noise'][n][0]
113 #print(noise)
113 #print(noise)
114 if self.CODE == 'spc_moments':
114 if self.CODE == 'spc_moments':
115 mean = data['moments'][n, 1]
115 mean = data['moments'][n, 1]
116 if ax.firsttime:
116 if ax.firsttime:
117 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
117 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
118 self.xmin = self.xmin if self.xmin else -self.xmax
118 self.xmin = self.xmin if self.xmin else -self.xmax
119 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
119 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
120 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
120 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
121 ax.plt = ax.pcolormesh(x, y, z[n].T,
121 ax.plt = ax.pcolormesh(x, y, z[n].T,
122 vmin=self.zmin,
122 vmin=self.zmin,
123 vmax=self.zmax,
123 vmax=self.zmax,
124 cmap=plt.get_cmap(self.colormap)
124 cmap=plt.get_cmap(self.colormap)
125 )
125 )
126
126
127 if self.showprofile:
127 if self.showprofile:
128 ax.plt_profile = self.pf_axes[n].plot(
128 ax.plt_profile = self.pf_axes[n].plot(
129 data['rti'][n], y)[0]
129 data['rti'][n], y)[0]
130 # ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
130 # ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
131 # color="k", linestyle="dashed", lw=1)[0]
131 # color="k", linestyle="dashed", lw=1)[0]
132 if self.CODE == 'spc_moments':
132 if self.CODE == 'spc_moments':
133 ax.plt_mean = ax.plot(mean, y, color='k')[0]
133 ax.plt_mean = ax.plot(mean, y, color='k')[0]
134 else:
134 else:
135 ax.plt.set_array(z[n].T.ravel())
135 ax.plt.set_array(z[n].T.ravel())
136 if self.showprofile:
136 if self.showprofile:
137 ax.plt_profile.set_data(data['rti'][n], y)
137 ax.plt_profile.set_data(data['rti'][n], y)
138 #ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
138 #ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
139 if self.CODE == 'spc_moments':
139 if self.CODE == 'spc_moments':
140 ax.plt_mean.set_data(mean, y)
140 ax.plt_mean.set_data(mean, y)
141 if len(self.azimuthList) > 0 and len(self.elevationList) > 0:
141 if len(self.azimuthList) > 0 and len(self.elevationList) > 0:
142 self.titles.append('CH {}: {:2.1f}elv {:2.1f}az {:3.2f}dB'.format(self.channelList[n], noise, self.elevationList[n], self.azimuthList[n]))
142 self.titles.append('CH {}: {:2.1f}elv {:2.1f}az {:3.2f}dB'.format(self.channelList[n], noise, self.elevationList[n], self.azimuthList[n]))
143 else:
143 else:
144 self.titles.append('CH {}: {:3.2f}dB'.format(self.channelList[n], noise))
144 self.titles.append('CH {}: {:3.2f}dB'.format(self.channelList[n], noise))
145
145
146
146
147 class CrossSpectraPlot(Plot):
147 class CrossSpectraPlot(Plot):
148
148
149 CODE = 'cspc'
149 CODE = 'cspc'
150 colormap = 'jet'
150 colormap = 'jet'
151 plot_type = 'pcolor'
151 plot_type = 'pcolor'
152 zmin_coh = None
152 zmin_coh = None
153 zmax_coh = None
153 zmax_coh = None
154 zmin_phase = None
154 zmin_phase = None
155 zmax_phase = None
155 zmax_phase = None
156 realChannels = None
156 realChannels = None
157 crossPairs = None
157 crossPairs = None
158
158
159 def setup(self):
159 def setup(self):
160
160
161 self.ncols = 4
161 self.ncols = 4
162 self.nplots = len(self.data.pairs) * 2
162 self.nplots = len(self.data.pairs) * 2
163 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
163 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
164 self.width = 3.1 * self.ncols
164 self.width = 3.1 * self.ncols
165 self.height = 2.6 * self.nrows
165 self.height = 2.6 * self.nrows
166 self.ylabel = 'Range [km]'
166 self.ylabel = 'Range [km]'
167 self.showprofile = False
167 self.showprofile = False
168 self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
168 self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
169
169
170 def update(self, dataOut):
170 def update(self, dataOut):
171
171
172 data = {}
172 data = {}
173 meta = {}
173 meta = {}
174
174
175 spc = dataOut.data_spc
175 spc = dataOut.data_spc
176 cspc = dataOut.data_cspc
176 cspc = dataOut.data_cspc
177 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
177 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
178 rawPairs = list(combinations(list(range(dataOut.nChannels)), 2))
178 rawPairs = list(combinations(list(range(dataOut.nChannels)), 2))
179 meta['pairs'] = rawPairs
179 meta['pairs'] = rawPairs
180
180
181 if self.crossPairs == None:
181 if self.crossPairs == None:
182 self.crossPairs = dataOut.pairsList
182 self.crossPairs = dataOut.pairsList
183
183
184 tmp = []
184 tmp = []
185
185
186 for n, pair in enumerate(meta['pairs']):
186 for n, pair in enumerate(meta['pairs']):
187
187
188 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
188 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
189 coh = numpy.abs(out)
189 coh = numpy.abs(out)
190 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
190 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
191 tmp.append(coh)
191 tmp.append(coh)
192 tmp.append(phase)
192 tmp.append(phase)
193
193
194 data['cspc'] = numpy.array(tmp)
194 data['cspc'] = numpy.array(tmp)
195
195
196 return data, meta
196 return data, meta
197
197
198 def plot(self):
198 def plot(self):
199
199
200 if self.xaxis == "frequency":
200 if self.xaxis == "frequency":
201 x = self.data.xrange[0]
201 x = self.data.xrange[0]
202 self.xlabel = "Frequency (kHz)"
202 self.xlabel = "Frequency (kHz)"
203 elif self.xaxis == "time":
203 elif self.xaxis == "time":
204 x = self.data.xrange[1]
204 x = self.data.xrange[1]
205 self.xlabel = "Time (ms)"
205 self.xlabel = "Time (ms)"
206 else:
206 else:
207 x = self.data.xrange[2]
207 x = self.data.xrange[2]
208 self.xlabel = "Velocity (m/s)"
208 self.xlabel = "Velocity (m/s)"
209
209
210 self.titles = []
210 self.titles = []
211
211
212 y = self.data.yrange
212 y = self.data.yrange
213 self.y = y
213 self.y = y
214
214
215 data = self.data[-1]
215 data = self.data[-1]
216 cspc = data['cspc']
216 cspc = data['cspc']
217
217
218 for n in range(len(self.data.pairs)):
218 for n in range(len(self.data.pairs)):
219
219
220 pair = self.crossPairs[n]
220 pair = self.crossPairs[n]
221
221
222 coh = cspc[n*2]
222 coh = cspc[n*2]
223 phase = cspc[n*2+1]
223 phase = cspc[n*2+1]
224 ax = self.axes[2 * n]
224 ax = self.axes[2 * n]
225
225
226 if ax.firsttime:
226 if ax.firsttime:
227 ax.plt = ax.pcolormesh(x, y, coh.T,
227 ax.plt = ax.pcolormesh(x, y, coh.T,
228 vmin=self.zmin_coh,
228 vmin=self.zmin_coh,
229 vmax=self.zmax_coh,
229 vmax=self.zmax_coh,
230 cmap=plt.get_cmap(self.colormap_coh)
230 cmap=plt.get_cmap(self.colormap_coh)
231 )
231 )
232 else:
232 else:
233 ax.plt.set_array(coh.T.ravel())
233 ax.plt.set_array(coh.T.ravel())
234 self.titles.append(
234 self.titles.append(
235 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
235 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
236
236
237 ax = self.axes[2 * n + 1]
237 ax = self.axes[2 * n + 1]
238 if ax.firsttime:
238 if ax.firsttime:
239 ax.plt = ax.pcolormesh(x, y, phase.T,
239 ax.plt = ax.pcolormesh(x, y, phase.T,
240 vmin=-180,
240 vmin=-180,
241 vmax=180,
241 vmax=180,
242 cmap=plt.get_cmap(self.colormap_phase)
242 cmap=plt.get_cmap(self.colormap_phase)
243 )
243 )
244 else:
244 else:
245 ax.plt.set_array(phase.T.ravel())
245 ax.plt.set_array(phase.T.ravel())
246
246
247 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
247 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
248
248
249
249
250 class RTIPlot(Plot):
250 class RTIPlot(Plot):
251 '''
251 '''
252 Plot for RTI data
252 Plot for RTI data
253 '''
253 '''
254
254
255 CODE = 'rti'
255 CODE = 'rti'
256 colormap = 'jet'
256 colormap = 'jet'
257 plot_type = 'pcolorbuffer'
257 plot_type = 'pcolorbuffer'
258 titles = None
258 titles = None
259 channelList = []
259 channelList = []
260 elevationList = []
260 elevationList = []
261 azimuthList = []
261 azimuthList = []
262
262
263 def setup(self):
263 def setup(self):
264 self.xaxis = 'time'
264 self.xaxis = 'time'
265 self.ncols = 1
265 self.ncols = 1
266 #print("dataChannels ",self.data.channels)
266 #print("dataChannels ",self.data.channels)
267 self.nrows = len(self.data.channels)
267 self.nrows = len(self.data.channels)
268 self.nplots = len(self.data.channels)
268 self.nplots = len(self.data.channels)
269 self.ylabel = 'Range [km]'
269 self.ylabel = 'Range [km]'
270 self.xlabel = 'Time'
270 self.xlabel = 'Time'
271 self.cb_label = 'dB'
271 self.cb_label = 'dB'
272 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
272 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
273 self.titles = ['{} Channel {}'.format(
273 self.titles = ['{} Channel {}'.format(
274 self.CODE.upper(), x) for x in range(self.nplots)]
274 self.CODE.upper(), x) for x in range(self.nplots)]
275
275
276 def update_list(self,dataOut):
276 def update_list(self,dataOut):
277
277
278 if len(self.channelList) == 0:
278 if len(self.channelList) == 0:
279 self.channelList = dataOut.channelList
279 self.channelList = dataOut.channelList
280 if len(self.elevationList) == 0:
280 if len(self.elevationList) == 0:
281 self.elevationList = dataOut.elevationList
281 self.elevationList = dataOut.elevationList
282 if len(self.azimuthList) == 0:
282 if len(self.azimuthList) == 0:
283 self.azimuthList = dataOut.azimuthList
283 self.azimuthList = dataOut.azimuthList
284
284
285
285
286 def update(self, dataOut):
286 def update(self, dataOut):
287 if len(self.channelList) == 0:
287 if len(self.channelList) == 0:
288 self.update_list(dataOut)
288 self.update_list(dataOut)
289 data = {}
289 data = {}
290 meta = {}
290 meta = {}
291
291
292 data['rti'] = dataOut.getPower()
292 data['rti'] = dataOut.getPower()
293
293
294 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
294 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
295 noise = 10*numpy.log10(dataOut.getNoise()/norm)
295 noise = 10*numpy.log10(dataOut.getNoise()/norm)
296 data['noise'] = noise
296 data['noise'] = noise
297
297
298 return data, meta
298 return data, meta
299
299
300 def plot(self):
300 def plot(self):
301
301
302 self.x = self.data.times
302 self.x = self.data.times
303 self.y = self.data.yrange
303 self.y = self.data.yrange
304 #print(" x, y: ",self.x, self.y)
304 #print(" x, y: ",self.x, self.y)
305 self.z = self.data[self.CODE]
305 self.z = self.data[self.CODE]
306 self.z = numpy.array(self.z, dtype=float)
306 self.z = numpy.array(self.z, dtype=float)
307 self.z = numpy.ma.masked_invalid(self.z)
307 self.z = numpy.ma.masked_invalid(self.z)
308
308
309 try:
309 try:
310 if self.channelList != None:
310 if self.channelList != None:
311 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
311 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
312 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
312 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
313 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
313 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
314 else:
314 else:
315 self.titles = ['{} Channel {}'.format(
315 self.titles = ['{} Channel {}'.format(
316 self.CODE.upper(), x) for x in self.channelList]
316 self.CODE.upper(), x) for x in self.channelList]
317 except:
317 except:
318 if self.channelList.any() != None:
318 if self.channelList.any() != None:
319
319 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
320 self.titles = ['{} Channel {}'.format(
320 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
321 self.CODE.upper(), x) for x in self.channelList]
321 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
322 else:
323 self.titles = ['{} Channel {}'.format(
324 self.CODE.upper(), x) for x in self.channelList]
322
325
323 if self.decimation is None:
326 if self.decimation is None:
324 x, y, z = self.fill_gaps(self.x, self.y, self.z)
327 x, y, z = self.fill_gaps(self.x, self.y, self.z)
325 else:
328 else:
326 x, y, z = self.fill_gaps(*self.decimate())
329 x, y, z = self.fill_gaps(*self.decimate())
327
330
328 #dummy_var = self.axes #Extrañamente esto actualiza el valor axes
331 #dummy_var = self.axes #Extrañamente esto actualiza el valor axes
329 for n, ax in enumerate(self.axes):
332 for n, ax in enumerate(self.axes):
330 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
333 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
331 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
334 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
332 data = self.data[-1]
335 data = self.data[-1]
333
336
334 if ax.firsttime:
337 if ax.firsttime:
335 ax.plt = ax.pcolormesh(x, y, z[n].T,
338 ax.plt = ax.pcolormesh(x, y, z[n].T,
336 vmin=self.zmin,
339 vmin=self.zmin,
337 vmax=self.zmax,
340 vmax=self.zmax,
338 cmap=plt.get_cmap(self.colormap)
341 cmap=plt.get_cmap(self.colormap)
339 )
342 )
340 if self.showprofile:
343 if self.showprofile:
341 ax.plot_profile = self.pf_axes[n].plot(data[self.CODE][n], self.y)[0]
344 ax.plot_profile = self.pf_axes[n].plot(data[self.CODE][n], self.y)[0]
342 if "noise" in self.data:
345 if "noise" in self.data:
343
346
344 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(data['noise'][n], len(self.y)), self.y,
347 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(data['noise'][n], len(self.y)), self.y,
345 color="k", linestyle="dashed", lw=1)[0]
348 color="k", linestyle="dashed", lw=1)[0]
346 else:
349 else:
347 ax.collections.remove(ax.collections[0])
350 ax.collections.remove(ax.collections[0])
348 ax.plt = ax.pcolormesh(x, y, z[n].T,
351 ax.plt = ax.pcolormesh(x, y, z[n].T,
349 vmin=self.zmin,
352 vmin=self.zmin,
350 vmax=self.zmax,
353 vmax=self.zmax,
351 cmap=plt.get_cmap(self.colormap)
354 cmap=plt.get_cmap(self.colormap)
352 )
355 )
353 if self.showprofile:
356 if self.showprofile:
354 ax.plot_profile.set_data(data[self.CODE][n], self.y)
357 ax.plot_profile.set_data(data[self.CODE][n], self.y)
355 if "noise" in self.data:
358 if "noise" in self.data:
356 ax.plot_noise.set_data(numpy.repeat(data['noise'][n], len(self.y)), self.y)
359 ax.plot_noise.set_data(numpy.repeat(data['noise'][n], len(self.y)), self.y)
357
360
358
361
359 class CoherencePlot(RTIPlot):
362 class CoherencePlot(RTIPlot):
360 '''
363 '''
361 Plot for Coherence data
364 Plot for Coherence data
362 '''
365 '''
363
366
364 CODE = 'coh'
367 CODE = 'coh'
365
368
366 def setup(self):
369 def setup(self):
367 self.xaxis = 'time'
370 self.xaxis = 'time'
368 self.ncols = 1
371 self.ncols = 1
369 self.nrows = len(self.data.pairs)
372 self.nrows = len(self.data.pairs)
370 self.nplots = len(self.data.pairs)
373 self.nplots = len(self.data.pairs)
371 self.ylabel = 'Range [km]'
374 self.ylabel = 'Range [km]'
372 self.xlabel = 'Time'
375 self.xlabel = 'Time'
373 self.plots_adjust.update({'hspace':0.6, 'left': 0.1, 'bottom': 0.1,'right':0.95})
376 self.plots_adjust.update({'hspace':0.6, 'left': 0.1, 'bottom': 0.1,'right':0.95})
374 if self.CODE == 'coh':
377 if self.CODE == 'coh':
375 self.cb_label = ''
378 self.cb_label = ''
376 self.titles = [
379 self.titles = [
377 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
380 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
378 else:
381 else:
379 self.cb_label = 'Degrees'
382 self.cb_label = 'Degrees'
380 self.titles = [
383 self.titles = [
381 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
384 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
382
385
383 def update(self, dataOut):
386 def update(self, dataOut):
384 self.update_list(dataOut)
387 self.update_list(dataOut)
385 data = {}
388 data = {}
386 meta = {}
389 meta = {}
387 data['coh'] = dataOut.getCoherence()
390 data['coh'] = dataOut.getCoherence()
388 meta['pairs'] = dataOut.pairsList
391 meta['pairs'] = dataOut.pairsList
389
392
390
393
391 return data, meta
394 return data, meta
392
395
393 class PhasePlot(CoherencePlot):
396 class PhasePlot(CoherencePlot):
394 '''
397 '''
395 Plot for Phase map data
398 Plot for Phase map data
396 '''
399 '''
397
400
398 CODE = 'phase'
401 CODE = 'phase'
399 colormap = 'seismic'
402 colormap = 'seismic'
400
403
401 def update(self, dataOut):
404 def update(self, dataOut):
402
405
403 data = {}
406 data = {}
404 meta = {}
407 meta = {}
405 data['phase'] = dataOut.getCoherence(phase=True)
408 data['phase'] = dataOut.getCoherence(phase=True)
406 meta['pairs'] = dataOut.pairsList
409 meta['pairs'] = dataOut.pairsList
407
410
408 return data, meta
411 return data, meta
409
412
410 class NoisePlot(Plot):
413 class NoisePlot(Plot):
411 '''
414 '''
412 Plot for noise
415 Plot for noise
413 '''
416 '''
414
417
415 CODE = 'noise'
418 CODE = 'noise'
416 plot_type = 'scatterbuffer'
419 plot_type = 'scatterbuffer'
417
420
418 def setup(self):
421 def setup(self):
419 self.xaxis = 'time'
422 self.xaxis = 'time'
420 self.ncols = 1
423 self.ncols = 1
421 self.nrows = 1
424 self.nrows = 1
422 self.nplots = 1
425 self.nplots = 1
423 self.ylabel = 'Intensity [dB]'
426 self.ylabel = 'Intensity [dB]'
424 self.xlabel = 'Time'
427 self.xlabel = 'Time'
425 self.titles = ['Noise']
428 self.titles = ['Noise']
426 self.colorbar = False
429 self.colorbar = False
427 self.plots_adjust.update({'right': 0.85 })
430 self.plots_adjust.update({'right': 0.85 })
428 #if not self.titles:
431 #if not self.titles:
429 self.titles = ['Noise Plot']
432 self.titles = ['Noise Plot']
430
433
431 def update(self, dataOut):
434 def update(self, dataOut):
432
435
433 data = {}
436 data = {}
434 meta = {}
437 meta = {}
435 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
438 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
436 noise = 10*numpy.log10(dataOut.getNoise())
439 noise = 10*numpy.log10(dataOut.getNoise())
437 noise = noise.reshape(dataOut.nChannels, 1)
440 noise = noise.reshape(dataOut.nChannels, 1)
438 data['noise'] = noise
441 data['noise'] = noise
439 meta['yrange'] = numpy.array([])
442 meta['yrange'] = numpy.array([])
440
443
441 return data, meta
444 return data, meta
442
445
443 def plot(self):
446 def plot(self):
444
447
445 x = self.data.times
448 x = self.data.times
446 xmin = self.data.min_time
449 xmin = self.data.min_time
447 xmax = xmin + self.xrange * 60 * 60
450 xmax = xmin + self.xrange * 60 * 60
448 Y = self.data['noise']
451 Y = self.data['noise']
449
452
450 if self.axes[0].firsttime:
453 if self.axes[0].firsttime:
451 if self.ymin is None: self.ymin = numpy.nanmin(Y) - 5
454 if self.ymin is None: self.ymin = numpy.nanmin(Y) - 5
452 if self.ymax is None: self.ymax = numpy.nanmax(Y) + 5
455 if self.ymax is None: self.ymax = numpy.nanmax(Y) + 5
453 for ch in self.data.channels:
456 for ch in self.data.channels:
454 y = Y[ch]
457 y = Y[ch]
455 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
458 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
456 plt.legend(bbox_to_anchor=(1.18, 1.0))
459 plt.legend(bbox_to_anchor=(1.18, 1.0))
457 else:
460 else:
458 for ch in self.data.channels:
461 for ch in self.data.channels:
459 y = Y[ch]
462 y = Y[ch]
460 self.axes[0].lines[ch].set_data(x, y)
463 self.axes[0].lines[ch].set_data(x, y)
461
464
462
465
463 class PowerProfilePlot(Plot):
466 class PowerProfilePlot(Plot):
464
467
465 CODE = 'pow_profile'
468 CODE = 'pow_profile'
466 plot_type = 'scatter'
469 plot_type = 'scatter'
467
470
468 def setup(self):
471 def setup(self):
469
472
470 self.ncols = 1
473 self.ncols = 1
471 self.nrows = 1
474 self.nrows = 1
472 self.nplots = 1
475 self.nplots = 1
473 self.height = 4
476 self.height = 4
474 self.width = 3
477 self.width = 3
475 self.ylabel = 'Range [km]'
478 self.ylabel = 'Range [km]'
476 self.xlabel = 'Intensity [dB]'
479 self.xlabel = 'Intensity [dB]'
477 self.titles = ['Power Profile']
480 self.titles = ['Power Profile']
478 self.colorbar = False
481 self.colorbar = False
479
482
480 def update(self, dataOut):
483 def update(self, dataOut):
481
484
482 data = {}
485 data = {}
483 meta = {}
486 meta = {}
484 data[self.CODE] = dataOut.getPower()
487 data[self.CODE] = dataOut.getPower()
485
488
486 return data, meta
489 return data, meta
487
490
488 def plot(self):
491 def plot(self):
489
492
490 y = self.data.yrange
493 y = self.data.yrange
491 self.y = y
494 self.y = y
492
495
493 x = self.data[-1][self.CODE]
496 x = self.data[-1][self.CODE]
494
497
495 if self.xmin is None: self.xmin = numpy.nanmin(x)*0.9
498 if self.xmin is None: self.xmin = numpy.nanmin(x)*0.9
496 if self.xmax is None: self.xmax = numpy.nanmax(x)*1.1
499 if self.xmax is None: self.xmax = numpy.nanmax(x)*1.1
497
500
498 if self.axes[0].firsttime:
501 if self.axes[0].firsttime:
499 for ch in self.data.channels:
502 for ch in self.data.channels:
500 self.axes[0].plot(x[ch], y, lw=1, label='Ch{}'.format(ch))
503 self.axes[0].plot(x[ch], y, lw=1, label='Ch{}'.format(ch))
501 plt.legend()
504 plt.legend()
502 else:
505 else:
503 for ch in self.data.channels:
506 for ch in self.data.channels:
504 self.axes[0].lines[ch].set_data(x[ch], y)
507 self.axes[0].lines[ch].set_data(x[ch], y)
505
508
506
509
507 class SpectraCutPlot(Plot):
510 class SpectraCutPlot(Plot):
508
511
509 CODE = 'spc_cut'
512 CODE = 'spc_cut'
510 plot_type = 'scatter'
513 plot_type = 'scatter'
511 buffering = False
514 buffering = False
512 heights = []
515 heights = []
513 channelList = []
516 channelList = []
514 maintitle = "Spectra Cuts"
517 maintitle = "Spectra Cuts"
515 flag_setIndex = False
518 flag_setIndex = False
516
519
517 def setup(self):
520 def setup(self):
518
521
519 self.nplots = len(self.data.channels)
522 self.nplots = len(self.data.channels)
520 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
523 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
521 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
524 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
522 self.width = 4.5 * self.ncols + 2.5
525 self.width = 4.5 * self.ncols + 2.5
523 self.height = 4.8 * self.nrows
526 self.height = 4.8 * self.nrows
524 self.ylabel = 'Power [dB]'
527 self.ylabel = 'Power [dB]'
525 self.colorbar = False
528 self.colorbar = False
526 self.plots_adjust.update({'left':0.1, 'hspace':0.3, 'right': 0.9, 'bottom':0.08})
529 self.plots_adjust.update({'left':0.1, 'hspace':0.3, 'right': 0.9, 'bottom':0.08})
527
530
528 if len(self.selectedHeightsList) > 0:
531 if len(self.selectedHeightsList) > 0:
529 self.maintitle = "Spectra Cut"# for %d km " %(int(self.selectedHeight))
532 self.maintitle = "Spectra Cut"# for %d km " %(int(self.selectedHeight))
530
533
531
534
532
535
533 def update(self, dataOut):
536 def update(self, dataOut):
534 if len(self.channelList) == 0:
537 if len(self.channelList) == 0:
535 self.channelList = dataOut.channelList
538 self.channelList = dataOut.channelList
536
539
537 self.heights = dataOut.heightList
540 self.heights = dataOut.heightList
538 #print("sels: ",self.selectedHeightsList)
541 #print("sels: ",self.selectedHeightsList)
539 if len(self.selectedHeightsList)>0 and not self.flag_setIndex:
542 if len(self.selectedHeightsList)>0 and not self.flag_setIndex:
540
543
541 for sel_height in self.selectedHeightsList:
544 for sel_height in self.selectedHeightsList:
542 index_list = numpy.where(self.heights >= sel_height)
545 index_list = numpy.where(self.heights >= sel_height)
543 index_list = index_list[0]
546 index_list = index_list[0]
544 self.height_index.append(index_list[0])
547 self.height_index.append(index_list[0])
545 #print("sels i:"", self.height_index)
548 #print("sels i:"", self.height_index)
546 self.flag_setIndex = True
549 self.flag_setIndex = True
547 #print(self.height_index)
550 #print(self.height_index)
548 data = {}
551 data = {}
549 meta = {}
552 meta = {}
550
553
551 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter#*dataOut.nFFTPoints
554 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter#*dataOut.nFFTPoints
552 n0 = 10*numpy.log10(dataOut.getNoise()/norm)
555 n0 = 10*numpy.log10(dataOut.getNoise()/norm)
553 noise = numpy.repeat(n0,(dataOut.nFFTPoints*dataOut.nHeights)).reshape(dataOut.nChannels,dataOut.nFFTPoints,dataOut.nHeights)
556 noise = numpy.repeat(n0,(dataOut.nFFTPoints*dataOut.nHeights)).reshape(dataOut.nChannels,dataOut.nFFTPoints,dataOut.nHeights)
554
557
555
558
556 z = []
559 z = []
557 for ch in range(dataOut.nChannels):
560 for ch in range(dataOut.nChannels):
558 if hasattr(dataOut.normFactor,'shape'):
561 if hasattr(dataOut.normFactor,'shape'):
559 z.append(numpy.divide(dataOut.data_spc[ch],dataOut.normFactor[ch]))
562 z.append(numpy.divide(dataOut.data_spc[ch],dataOut.normFactor[ch]))
560 else:
563 else:
561 z.append(numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
564 z.append(numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
562
565
563 z = numpy.asarray(z)
566 z = numpy.asarray(z)
564 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
567 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
565 spc = 10*numpy.log10(z)
568 spc = 10*numpy.log10(z)
566
569
567
570
568 data['spc'] = spc - noise
571 data['spc'] = spc - noise
569 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
572 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
570
573
571 return data, meta
574 return data, meta
572
575
573 def plot(self):
576 def plot(self):
574 if self.xaxis == "frequency":
577 if self.xaxis == "frequency":
575 x = self.data.xrange[0][1:]
578 x = self.data.xrange[0][1:]
576 self.xlabel = "Frequency (kHz)"
579 self.xlabel = "Frequency (kHz)"
577 elif self.xaxis == "time":
580 elif self.xaxis == "time":
578 x = self.data.xrange[1]
581 x = self.data.xrange[1]
579 self.xlabel = "Time (ms)"
582 self.xlabel = "Time (ms)"
580 else:
583 else:
581 x = self.data.xrange[2]
584 x = self.data.xrange[2]
582 self.xlabel = "Velocity (m/s)"
585 self.xlabel = "Velocity (m/s)"
583
586
584 self.titles = []
587 self.titles = []
585
588
586 y = self.data.yrange
589 y = self.data.yrange
587 z = self.data[-1]['spc']
590 z = self.data[-1]['spc']
588 #print(z.shape)
591 #print(z.shape)
589 if len(self.height_index) > 0:
592 if len(self.height_index) > 0:
590 index = self.height_index
593 index = self.height_index
591 else:
594 else:
592 index = numpy.arange(0, len(y), int((len(y))/9))
595 index = numpy.arange(0, len(y), int((len(y))/9))
593 #print("inde x ", index, self.axes)
596 #print("inde x ", index, self.axes)
594
597
595 for n, ax in enumerate(self.axes):
598 for n, ax in enumerate(self.axes):
596
599
597 if ax.firsttime:
600 if ax.firsttime:
598
601
599
602
600 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
603 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
601 self.xmin = self.xmin if self.xmin else -self.xmax
604 self.xmin = self.xmin if self.xmin else -self.xmax
602 self.ymin = self.ymin if self.ymin else numpy.nanmin(z)
605 self.ymin = self.ymin if self.ymin else numpy.nanmin(z)
603 self.ymax = self.ymax if self.ymax else numpy.nanmax(z)
606 self.ymax = self.ymax if self.ymax else numpy.nanmax(z)
604
607
605
608
606 ax.plt = ax.plot(x, z[n, :, index].T)
609 ax.plt = ax.plot(x, z[n, :, index].T)
607 labels = ['Range = {:2.1f}km'.format(y[i]) for i in index]
610 labels = ['Range = {:2.1f}km'.format(y[i]) for i in index]
608 self.figures[0].legend(ax.plt, labels, loc='center right', prop={'size': 8})
611 self.figures[0].legend(ax.plt, labels, loc='center right', prop={'size': 8})
609 ax.minorticks_on()
612 ax.minorticks_on()
610 ax.grid(which='major', axis='both')
613 ax.grid(which='major', axis='both')
611 ax.grid(which='minor', axis='x')
614 ax.grid(which='minor', axis='x')
612 else:
615 else:
613 for i, line in enumerate(ax.plt):
616 for i, line in enumerate(ax.plt):
614 line.set_data(x, z[n, :, index[i]])
617 line.set_data(x, z[n, :, index[i]])
615
618
616
619
617 self.titles.append('CH {}'.format(self.channelList[n]))
620 self.titles.append('CH {}'.format(self.channelList[n]))
618 plt.suptitle(self.maintitle, fontsize=10)
621 plt.suptitle(self.maintitle, fontsize=10)
619
622
620
623
621 class BeaconPhase(Plot):
624 class BeaconPhase(Plot):
622
625
623 __isConfig = None
626 __isConfig = None
624 __nsubplots = None
627 __nsubplots = None
625
628
626 PREFIX = 'beacon_phase'
629 PREFIX = 'beacon_phase'
627
630
628 def __init__(self):
631 def __init__(self):
629 Plot.__init__(self)
632 Plot.__init__(self)
630 self.timerange = 24*60*60
633 self.timerange = 24*60*60
631 self.isConfig = False
634 self.isConfig = False
632 self.__nsubplots = 1
635 self.__nsubplots = 1
633 self.counter_imagwr = 0
636 self.counter_imagwr = 0
634 self.WIDTH = 800
637 self.WIDTH = 800
635 self.HEIGHT = 400
638 self.HEIGHT = 400
636 self.WIDTHPROF = 120
639 self.WIDTHPROF = 120
637 self.HEIGHTPROF = 0
640 self.HEIGHTPROF = 0
638 self.xdata = None
641 self.xdata = None
639 self.ydata = None
642 self.ydata = None
640
643
641 self.PLOT_CODE = BEACON_CODE
644 self.PLOT_CODE = BEACON_CODE
642
645
643 self.FTP_WEI = None
646 self.FTP_WEI = None
644 self.EXP_CODE = None
647 self.EXP_CODE = None
645 self.SUB_EXP_CODE = None
648 self.SUB_EXP_CODE = None
646 self.PLOT_POS = None
649 self.PLOT_POS = None
647
650
648 self.filename_phase = None
651 self.filename_phase = None
649
652
650 self.figfile = None
653 self.figfile = None
651
654
652 self.xmin = None
655 self.xmin = None
653 self.xmax = None
656 self.xmax = None
654
657
655 def getSubplots(self):
658 def getSubplots(self):
656
659
657 ncol = 1
660 ncol = 1
658 nrow = 1
661 nrow = 1
659
662
660 return nrow, ncol
663 return nrow, ncol
661
664
662 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
665 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
663
666
664 self.__showprofile = showprofile
667 self.__showprofile = showprofile
665 self.nplots = nplots
668 self.nplots = nplots
666
669
667 ncolspan = 7
670 ncolspan = 7
668 colspan = 6
671 colspan = 6
669 self.__nsubplots = 2
672 self.__nsubplots = 2
670
673
671 self.createFigure(id = id,
674 self.createFigure(id = id,
672 wintitle = wintitle,
675 wintitle = wintitle,
673 widthplot = self.WIDTH+self.WIDTHPROF,
676 widthplot = self.WIDTH+self.WIDTHPROF,
674 heightplot = self.HEIGHT+self.HEIGHTPROF,
677 heightplot = self.HEIGHT+self.HEIGHTPROF,
675 show=show)
678 show=show)
676
679
677 nrow, ncol = self.getSubplots()
680 nrow, ncol = self.getSubplots()
678
681
679 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
682 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
680
683
681 def save_phase(self, filename_phase):
684 def save_phase(self, filename_phase):
682 f = open(filename_phase,'w+')
685 f = open(filename_phase,'w+')
683 f.write('\n\n')
686 f.write('\n\n')
684 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
687 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
685 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
688 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
686 f.close()
689 f.close()
687
690
688 def save_data(self, filename_phase, data, data_datetime):
691 def save_data(self, filename_phase, data, data_datetime):
689 f=open(filename_phase,'a')
692 f=open(filename_phase,'a')
690 timetuple_data = data_datetime.timetuple()
693 timetuple_data = data_datetime.timetuple()
691 day = str(timetuple_data.tm_mday)
694 day = str(timetuple_data.tm_mday)
692 month = str(timetuple_data.tm_mon)
695 month = str(timetuple_data.tm_mon)
693 year = str(timetuple_data.tm_year)
696 year = str(timetuple_data.tm_year)
694 hour = str(timetuple_data.tm_hour)
697 hour = str(timetuple_data.tm_hour)
695 minute = str(timetuple_data.tm_min)
698 minute = str(timetuple_data.tm_min)
696 second = str(timetuple_data.tm_sec)
699 second = str(timetuple_data.tm_sec)
697 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
700 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
698 f.close()
701 f.close()
699
702
700 def plot(self):
703 def plot(self):
701 log.warning('TODO: Not yet implemented...')
704 log.warning('TODO: Not yet implemented...')
702
705
703 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
706 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
704 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
707 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
705 timerange=None,
708 timerange=None,
706 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
709 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
707 server=None, folder=None, username=None, password=None,
710 server=None, folder=None, username=None, password=None,
708 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
711 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
709
712
710 if dataOut.flagNoData:
713 if dataOut.flagNoData:
711 return dataOut
714 return dataOut
712
715
713 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
716 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
714 return
717 return
715
718
716 if pairsList == None:
719 if pairsList == None:
717 pairsIndexList = dataOut.pairsIndexList[:10]
720 pairsIndexList = dataOut.pairsIndexList[:10]
718 else:
721 else:
719 pairsIndexList = []
722 pairsIndexList = []
720 for pair in pairsList:
723 for pair in pairsList:
721 if pair not in dataOut.pairsList:
724 if pair not in dataOut.pairsList:
722 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
725 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
723 pairsIndexList.append(dataOut.pairsList.index(pair))
726 pairsIndexList.append(dataOut.pairsList.index(pair))
724
727
725 if pairsIndexList == []:
728 if pairsIndexList == []:
726 return
729 return
727
730
728 # if len(pairsIndexList) > 4:
731 # if len(pairsIndexList) > 4:
729 # pairsIndexList = pairsIndexList[0:4]
732 # pairsIndexList = pairsIndexList[0:4]
730
733
731 hmin_index = None
734 hmin_index = None
732 hmax_index = None
735 hmax_index = None
733
736
734 if hmin != None and hmax != None:
737 if hmin != None and hmax != None:
735 indexes = numpy.arange(dataOut.nHeights)
738 indexes = numpy.arange(dataOut.nHeights)
736 hmin_list = indexes[dataOut.heightList >= hmin]
739 hmin_list = indexes[dataOut.heightList >= hmin]
737 hmax_list = indexes[dataOut.heightList <= hmax]
740 hmax_list = indexes[dataOut.heightList <= hmax]
738
741
739 if hmin_list.any():
742 if hmin_list.any():
740 hmin_index = hmin_list[0]
743 hmin_index = hmin_list[0]
741
744
742 if hmax_list.any():
745 if hmax_list.any():
743 hmax_index = hmax_list[-1]+1
746 hmax_index = hmax_list[-1]+1
744
747
745 x = dataOut.getTimeRange()
748 x = dataOut.getTimeRange()
746
749
747 thisDatetime = dataOut.datatime
750 thisDatetime = dataOut.datatime
748
751
749 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
752 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
750 xlabel = "Local Time"
753 xlabel = "Local Time"
751 ylabel = "Phase (degrees)"
754 ylabel = "Phase (degrees)"
752
755
753 update_figfile = False
756 update_figfile = False
754
757
755 nplots = len(pairsIndexList)
758 nplots = len(pairsIndexList)
756 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
759 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
757 phase_beacon = numpy.zeros(len(pairsIndexList))
760 phase_beacon = numpy.zeros(len(pairsIndexList))
758 for i in range(nplots):
761 for i in range(nplots):
759 pair = dataOut.pairsList[pairsIndexList[i]]
762 pair = dataOut.pairsList[pairsIndexList[i]]
760 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
763 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
761 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
764 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
762 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
765 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
763 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
766 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
764 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
767 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
765
768
766 if dataOut.beacon_heiIndexList:
769 if dataOut.beacon_heiIndexList:
767 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
770 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
768 else:
771 else:
769 phase_beacon[i] = numpy.average(phase)
772 phase_beacon[i] = numpy.average(phase)
770
773
771 if not self.isConfig:
774 if not self.isConfig:
772
775
773 nplots = len(pairsIndexList)
776 nplots = len(pairsIndexList)
774
777
775 self.setup(id=id,
778 self.setup(id=id,
776 nplots=nplots,
779 nplots=nplots,
777 wintitle=wintitle,
780 wintitle=wintitle,
778 showprofile=showprofile,
781 showprofile=showprofile,
779 show=show)
782 show=show)
780
783
781 if timerange != None:
784 if timerange != None:
782 self.timerange = timerange
785 self.timerange = timerange
783
786
784 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
787 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
785
788
786 if ymin == None: ymin = 0
789 if ymin == None: ymin = 0
787 if ymax == None: ymax = 360
790 if ymax == None: ymax = 360
788
791
789 self.FTP_WEI = ftp_wei
792 self.FTP_WEI = ftp_wei
790 self.EXP_CODE = exp_code
793 self.EXP_CODE = exp_code
791 self.SUB_EXP_CODE = sub_exp_code
794 self.SUB_EXP_CODE = sub_exp_code
792 self.PLOT_POS = plot_pos
795 self.PLOT_POS = plot_pos
793
796
794 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
797 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
795 self.isConfig = True
798 self.isConfig = True
796 self.figfile = figfile
799 self.figfile = figfile
797 self.xdata = numpy.array([])
800 self.xdata = numpy.array([])
798 self.ydata = numpy.array([])
801 self.ydata = numpy.array([])
799
802
800 update_figfile = True
803 update_figfile = True
801
804
802 #open file beacon phase
805 #open file beacon phase
803 path = '%s%03d' %(self.PREFIX, self.id)
806 path = '%s%03d' %(self.PREFIX, self.id)
804 beacon_file = os.path.join(path,'%s.txt'%self.name)
807 beacon_file = os.path.join(path,'%s.txt'%self.name)
805 self.filename_phase = os.path.join(figpath,beacon_file)
808 self.filename_phase = os.path.join(figpath,beacon_file)
806 #self.save_phase(self.filename_phase)
809 #self.save_phase(self.filename_phase)
807
810
808
811
809 #store data beacon phase
812 #store data beacon phase
810 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
813 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
811
814
812 self.setWinTitle(title)
815 self.setWinTitle(title)
813
816
814
817
815 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
818 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
816
819
817 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
820 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
818
821
819 axes = self.axesList[0]
822 axes = self.axesList[0]
820
823
821 self.xdata = numpy.hstack((self.xdata, x[0:1]))
824 self.xdata = numpy.hstack((self.xdata, x[0:1]))
822
825
823 if len(self.ydata)==0:
826 if len(self.ydata)==0:
824 self.ydata = phase_beacon.reshape(-1,1)
827 self.ydata = phase_beacon.reshape(-1,1)
825 else:
828 else:
826 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
829 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
827
830
828
831
829 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
832 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
830 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
833 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
831 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
834 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
832 XAxisAsTime=True, grid='both'
835 XAxisAsTime=True, grid='both'
833 )
836 )
834
837
835 self.draw()
838 self.draw()
836
839
837 if dataOut.ltctime >= self.xmax:
840 if dataOut.ltctime >= self.xmax:
838 self.counter_imagwr = wr_period
841 self.counter_imagwr = wr_period
839 self.isConfig = False
842 self.isConfig = False
840 update_figfile = True
843 update_figfile = True
841
844
842 self.save(figpath=figpath,
845 self.save(figpath=figpath,
843 figfile=figfile,
846 figfile=figfile,
844 save=save,
847 save=save,
845 ftp=ftp,
848 ftp=ftp,
846 wr_period=wr_period,
849 wr_period=wr_period,
847 thisDatetime=thisDatetime,
850 thisDatetime=thisDatetime,
848 update_figfile=update_figfile)
851 update_figfile=update_figfile)
849
852
850 return dataOut
853 return dataOut
851
854
852 class NoiselessSpectraPlot(Plot):
855 class NoiselessSpectraPlot(Plot):
853 '''
856 '''
854 Plot for Spectra data, subtracting
857 Plot for Spectra data, subtracting
855 the noise in all channels, using for
858 the noise in all channels, using for
856 amisr-14 data
859 amisr-14 data
857 '''
860 '''
858
861
859 CODE = 'noiseless_spc'
862 CODE = 'noiseless_spc'
860 colormap = 'jet'
863 colormap = 'jet'
861 plot_type = 'pcolor'
864 plot_type = 'pcolor'
862 buffering = False
865 buffering = False
863 channelList = []
866 channelList = []
864 last_noise = None
867 last_noise = None
865
868
866 def setup(self):
869 def setup(self):
867
870
868 self.nplots = len(self.data.channels)
871 self.nplots = len(self.data.channels)
869 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
872 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
870 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
873 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
871 self.height = 3.5 * self.nrows
874 self.height = 3.5 * self.nrows
872
875
873 self.cb_label = 'dB'
876 self.cb_label = 'dB'
874 if self.showprofile:
877 if self.showprofile:
875 self.width = 5.8 * self.ncols
878 self.width = 5.8 * self.ncols
876 else:
879 else:
877 self.width = 4.8* self.ncols
880 self.width = 4.8* self.ncols
878 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.92, 'bottom': 0.12})
881 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.92, 'bottom': 0.12})
879
882
880 self.ylabel = 'Range [km]'
883 self.ylabel = 'Range [km]'
881
884
882
885
883 def update_list(self,dataOut):
886 def update_list(self,dataOut):
884 if len(self.channelList) == 0:
887 if len(self.channelList) == 0:
885 self.channelList = dataOut.channelList
888 self.channelList = dataOut.channelList
886
889
887 def update(self, dataOut):
890 def update(self, dataOut):
888
891
889 self.update_list(dataOut)
892 self.update_list(dataOut)
890 data = {}
893 data = {}
891 meta = {}
894 meta = {}
892
895
893 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter#*dataOut.nFFTPoints
896 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter#*dataOut.nFFTPoints
894 n0 = 10*numpy.log10(dataOut.getNoise()/norm)
897 n0 = 10*numpy.log10(dataOut.getNoise()/norm)
895 noise = numpy.repeat(n0,(dataOut.nFFTPoints*dataOut.nHeights)).reshape(dataOut.nChannels,dataOut.nFFTPoints,dataOut.nHeights)
898 noise = numpy.repeat(n0,(dataOut.nFFTPoints*dataOut.nHeights)).reshape(dataOut.nChannels,dataOut.nFFTPoints,dataOut.nHeights)
896
899
897 z = []
900 z = []
898 for ch in range(dataOut.nChannels):
901 for ch in range(dataOut.nChannels):
899 if hasattr(dataOut.normFactor,'shape'):
902 if hasattr(dataOut.normFactor,'shape'):
900 z.append(numpy.divide(dataOut.data_spc[ch],dataOut.normFactor[ch]))
903 z.append(numpy.divide(dataOut.data_spc[ch],dataOut.normFactor[ch]))
901 else:
904 else:
902 z.append(numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
905 z.append(numpy.divide(dataOut.data_spc[ch],dataOut.normFactor))
903
906
904 z = numpy.asarray(z)
907 z = numpy.asarray(z)
905 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
908 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
906 spc = 10*numpy.log10(z)
909 spc = 10*numpy.log10(z)
907
910
908
911
909 data['spc'] = spc - noise
912 data['spc'] = spc - noise
910 #print(spc.shape)
913 #print(spc.shape)
911 data['rti'] = spc.mean(axis=1)
914 data['rti'] = spc.mean(axis=1)
912 data['noise'] = noise
915 data['noise'] = noise
913
916
914
917
915
918
916 # data['noise'] = noise
919 # data['noise'] = noise
917 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
920 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
918
921
919 return data, meta
922 return data, meta
920
923
921 def plot(self):
924 def plot(self):
922 if self.xaxis == "frequency":
925 if self.xaxis == "frequency":
923 x = self.data.xrange[0]
926 x = self.data.xrange[0]
924 self.xlabel = "Frequency (kHz)"
927 self.xlabel = "Frequency (kHz)"
925 elif self.xaxis == "time":
928 elif self.xaxis == "time":
926 x = self.data.xrange[1]
929 x = self.data.xrange[1]
927 self.xlabel = "Time (ms)"
930 self.xlabel = "Time (ms)"
928 else:
931 else:
929 x = self.data.xrange[2]
932 x = self.data.xrange[2]
930 self.xlabel = "Velocity (m/s)"
933 self.xlabel = "Velocity (m/s)"
931
934
932 self.titles = []
935 self.titles = []
933 y = self.data.yrange
936 y = self.data.yrange
934 self.y = y
937 self.y = y
935
938
936 data = self.data[-1]
939 data = self.data[-1]
937 z = data['spc']
940 z = data['spc']
938
941
939 for n, ax in enumerate(self.axes):
942 for n, ax in enumerate(self.axes):
940 #noise = data['noise'][n]
943 #noise = data['noise'][n]
941
944
942 if ax.firsttime:
945 if ax.firsttime:
943 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
946 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
944 self.xmin = self.xmin if self.xmin else -self.xmax
947 self.xmin = self.xmin if self.xmin else -self.xmax
945 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
948 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
946 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
949 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
947 ax.plt = ax.pcolormesh(x, y, z[n].T,
950 ax.plt = ax.pcolormesh(x, y, z[n].T,
948 vmin=self.zmin,
951 vmin=self.zmin,
949 vmax=self.zmax,
952 vmax=self.zmax,
950 cmap=plt.get_cmap(self.colormap)
953 cmap=plt.get_cmap(self.colormap)
951 )
954 )
952
955
953 if self.showprofile:
956 if self.showprofile:
954 ax.plt_profile = self.pf_axes[n].plot(
957 ax.plt_profile = self.pf_axes[n].plot(
955 data['rti'][n], y)[0]
958 data['rti'][n], y)[0]
956
959
957
960
958 else:
961 else:
959 ax.plt.set_array(z[n].T.ravel())
962 ax.plt.set_array(z[n].T.ravel())
960 if self.showprofile:
963 if self.showprofile:
961 ax.plt_profile.set_data(data['rti'][n], y)
964 ax.plt_profile.set_data(data['rti'][n], y)
962
965
963
966
964 self.titles.append('CH {}'.format(self.channelList[n]))
967 self.titles.append('CH {}'.format(self.channelList[n]))
965
968
966
969
967 class NoiselessRTIPlot(Plot):
970 class NoiselessRTIPlot(Plot):
968 '''
971 '''
969 Plot for RTI data
972 Plot for RTI data
970 '''
973 '''
971
974
972 CODE = 'noiseless_rti'
975 CODE = 'noiseless_rti'
973 colormap = 'jet'
976 colormap = 'jet'
974 plot_type = 'pcolorbuffer'
977 plot_type = 'pcolorbuffer'
975 titles = None
978 titles = None
976 channelList = []
979 channelList = []
977 elevationList = []
980 elevationList = []
978 azimuthList = []
981 azimuthList = []
979 last_noise = None
982 last_noise = None
980
983
981 def setup(self):
984 def setup(self):
982 self.xaxis = 'time'
985 self.xaxis = 'time'
983 self.ncols = 1
986 self.ncols = 1
984 #print("dataChannels ",self.data.channels)
987 #print("dataChannels ",self.data.channels)
985 self.nrows = len(self.data.channels)
988 self.nrows = len(self.data.channels)
986 self.nplots = len(self.data.channels)
989 self.nplots = len(self.data.channels)
987 self.ylabel = 'Range [km]'
990 self.ylabel = 'Range [km]'
988 self.xlabel = 'Time'
991 self.xlabel = 'Time'
989 self.cb_label = 'dB'
992 self.cb_label = 'dB'
990 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
993 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
991 self.titles = ['{} Channel {}'.format(
994 self.titles = ['{} Channel {}'.format(
992 self.CODE.upper(), x) for x in range(self.nplots)]
995 self.CODE.upper(), x) for x in range(self.nplots)]
993
996
994 def update_list(self,dataOut):
997 def update_list(self,dataOut):
995 if len(self.channelList) == 0:
998 if len(self.channelList) == 0:
996 self.channelList = dataOut.channelList
999 self.channelList = dataOut.channelList
997 if len(self.elevationList) == 0:
1000 if len(self.elevationList) == 0:
998 self.elevationList = dataOut.elevationList
1001 self.elevationList = dataOut.elevationList
999 if len(self.azimuthList) == 0:
1002 if len(self.azimuthList) == 0:
1000 self.azimuthList = dataOut.azimuthList
1003 self.azimuthList = dataOut.azimuthList
1001
1004
1002 def update(self, dataOut):
1005 def update(self, dataOut):
1003 if len(self.channelList) == 0:
1006 if len(self.channelList) == 0:
1004 self.update_list(dataOut)
1007 self.update_list(dataOut)
1008
1005 data = {}
1009 data = {}
1006 meta = {}
1010 meta = {}
1007 #print(dataOut.max_nIncohInt, dataOut.nIncohInt)
1011 #print(dataOut.max_nIncohInt, dataOut.nIncohInt)
1008 #print(dataOut.windowOfFilter,dataOut.nCohInt,dataOut.nProfiles,dataOut.max_nIncohInt,dataOut.nIncohInt)
1012 #print(dataOut.windowOfFilter,dataOut.nCohInt,dataOut.nProfiles,dataOut.max_nIncohInt,dataOut.nIncohInt)
1009 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
1013 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
1010
1014
1011
1015
1012 n0 = 10*numpy.log10(dataOut.getNoise()/norm)
1016 n0 = 10*numpy.log10(dataOut.getNoise()/norm)
1013
1017
1014 data['noise'] = n0
1018 data['noise'] = n0
1015 noise = numpy.repeat(n0,dataOut.nHeights).reshape(dataOut.nChannels,dataOut.nHeights)
1019 noise = numpy.repeat(n0,dataOut.nHeights).reshape(dataOut.nChannels,dataOut.nHeights)
1016
1020
1017 data['noiseless_rti'] = dataOut.getPower() - noise
1021 data['noiseless_rti'] = dataOut.getPower() - noise
1018
1022
1019 return data, meta
1023 return data, meta
1020
1024
1021 def plot(self):
1025 def plot(self):
1022
1026
1023 self.x = self.data.times
1027 self.x = self.data.times
1024 self.y = self.data.yrange
1028 self.y = self.data.yrange
1025 self.z = self.data['noiseless_rti']
1029 self.z = self.data['noiseless_rti']
1026 self.z = numpy.array(self.z, dtype=float)
1030 self.z = numpy.array(self.z, dtype=float)
1027 self.z = numpy.ma.masked_invalid(self.z)
1031 self.z = numpy.ma.masked_invalid(self.z)
1028
1032
1029
1033
1030 try:
1034 try:
1031 if self.channelList != None:
1035 if self.channelList != None:
1032 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
1036 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
1033 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
1037 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
1034 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
1038 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
1035 else:
1039 else:
1036 self.titles = ['{} Channel {}'.format(
1040 self.titles = ['{} Channel {}'.format(
1037 self.CODE.upper(), x) for x in self.channelList]
1041 self.CODE.upper(), x) for x in self.channelList]
1038 except:
1042 except:
1039 if self.channelList.any() != None:
1043 if self.channelList.any() != None:
1040
1044 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
1041 self.titles = ['{} Channel {}'.format(
1045 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
1042 self.CODE.upper(), x) for x in self.channelList]
1046 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
1047 else:
1048 self.titles = ['{} Channel {}'.format(
1049 self.CODE.upper(), x) for x in self.channelList]
1043
1050
1044
1051
1045 if self.decimation is None:
1052 if self.decimation is None:
1046 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1053 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1047 else:
1054 else:
1048 x, y, z = self.fill_gaps(*self.decimate())
1055 x, y, z = self.fill_gaps(*self.decimate())
1049 dummy_var = self.axes #Extrañamente esto actualiza el valor axes
1056 dummy_var = self.axes #Extrañamente esto actualiza el valor axes
1050 #print("plot shapes ", z.shape, x.shape, y.shape)
1057 #print("plot shapes ", z.shape, x.shape, y.shape)
1051 for n, ax in enumerate(self.axes):
1058 for n, ax in enumerate(self.axes):
1052
1059
1053
1060
1054 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
1061 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
1055 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
1062 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
1056 data = self.data[-1]
1063 data = self.data[-1]
1057 if ax.firsttime:
1064 if ax.firsttime:
1058 ax.plt = ax.pcolormesh(x, y, z[n].T,
1065 ax.plt = ax.pcolormesh(x, y, z[n].T,
1059 vmin=self.zmin,
1066 vmin=self.zmin,
1060 vmax=self.zmax,
1067 vmax=self.zmax,
1061 cmap=plt.get_cmap(self.colormap)
1068 cmap=plt.get_cmap(self.colormap)
1062 )
1069 )
1063 if self.showprofile:
1070 if self.showprofile:
1064 ax.plot_profile = self.pf_axes[n].plot(data['noiseless_rti'][n], self.y)[0]
1071 ax.plot_profile = self.pf_axes[n].plot(data['noiseless_rti'][n], self.y)[0]
1065
1072
1066 else:
1073 else:
1067 ax.collections.remove(ax.collections[0])
1074 ax.collections.remove(ax.collections[0])
1068 ax.plt = ax.pcolormesh(x, y, z[n].T,
1075 ax.plt = ax.pcolormesh(x, y, z[n].T,
1069 vmin=self.zmin,
1076 vmin=self.zmin,
1070 vmax=self.zmax,
1077 vmax=self.zmax,
1071 cmap=plt.get_cmap(self.colormap)
1078 cmap=plt.get_cmap(self.colormap)
1072 )
1079 )
1073 if self.showprofile:
1080 if self.showprofile:
1074 ax.plot_profile.set_data(data['noiseless_rti'][n], self.y)
1081 ax.plot_profile.set_data(data['noiseless_rti'][n], self.y)
1075 # if "noise" in self.data:
1082 # if "noise" in self.data:
1076 # #ax.plot_noise.set_data(numpy.repeat(data['noise'][n], len(self.y)), self.y)
1083 # #ax.plot_noise.set_data(numpy.repeat(data['noise'][n], len(self.y)), self.y)
1077 # ax.plot_noise.set_data(data['noise'][n], self.y)
1084 # ax.plot_noise.set_data(data['noise'][n], self.y)
1078
1085
1079
1086
1080 class OutliersRTIPlot(Plot):
1087 class OutliersRTIPlot(Plot):
1081 '''
1088 '''
1082 Plot for data_xxxx object
1089 Plot for data_xxxx object
1083 '''
1090 '''
1084
1091
1085 CODE = 'outlier_rtc' # Range Time Counts
1092 CODE = 'outlier_rtc' # Range Time Counts
1086 colormap = 'cool'
1093 colormap = 'cool'
1087 plot_type = 'pcolorbuffer'
1094 plot_type = 'pcolorbuffer'
1088
1095
1089 def setup(self):
1096 def setup(self):
1090 self.xaxis = 'time'
1097 self.xaxis = 'time'
1091 self.ncols = 1
1098 self.ncols = 1
1092 self.nrows = self.data.shape('outlier_rtc')[0]
1099 self.nrows = self.data.shape('outlier_rtc')[0]
1093 self.nplots = self.nrows
1100 self.nplots = self.nrows
1094 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
1101 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
1095
1102
1096
1103
1097 if not self.xlabel:
1104 if not self.xlabel:
1098 self.xlabel = 'Time'
1105 self.xlabel = 'Time'
1099
1106
1100 self.ylabel = 'Height [km]'
1107 self.ylabel = 'Height [km]'
1101 if not self.titles:
1108 if not self.titles:
1102 self.titles = ['Outliers Ch:{}'.format(x) for x in range(self.nrows)]
1109 self.titles = ['Outliers Ch:{}'.format(x) for x in range(self.nrows)]
1103
1110
1104 def update(self, dataOut):
1111 def update(self, dataOut):
1105
1112
1106 data = {}
1113 data = {}
1107 data['outlier_rtc'] = dataOut.data_outlier
1114 data['outlier_rtc'] = dataOut.data_outlier
1108
1115
1109 meta = {}
1116 meta = {}
1110
1117
1111 return data, meta
1118 return data, meta
1112
1119
1113 def plot(self):
1120 def plot(self):
1114 # self.data.normalize_heights()
1121 # self.data.normalize_heights()
1115 self.x = self.data.times
1122 self.x = self.data.times
1116 self.y = self.data.yrange
1123 self.y = self.data.yrange
1117 self.z = self.data['outlier_rtc']
1124 self.z = self.data['outlier_rtc']
1118
1125
1119 #self.z = numpy.ma.masked_invalid(self.z)
1126 #self.z = numpy.ma.masked_invalid(self.z)
1120
1127
1121 if self.decimation is None:
1128 if self.decimation is None:
1122 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1129 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1123 else:
1130 else:
1124 x, y, z = self.fill_gaps(*self.decimate())
1131 x, y, z = self.fill_gaps(*self.decimate())
1125
1132
1126 for n, ax in enumerate(self.axes):
1133 for n, ax in enumerate(self.axes):
1127
1134
1128 self.zmax = self.zmax if self.zmax is not None else numpy.max(
1135 self.zmax = self.zmax if self.zmax is not None else numpy.max(
1129 self.z[n])
1136 self.z[n])
1130 self.zmin = self.zmin if self.zmin is not None else numpy.min(
1137 self.zmin = self.zmin if self.zmin is not None else numpy.min(
1131 self.z[n])
1138 self.z[n])
1132 data = self.data[-1]
1139 data = self.data[-1]
1133 if ax.firsttime:
1140 if ax.firsttime:
1134 if self.zlimits is not None:
1141 if self.zlimits is not None:
1135 self.zmin, self.zmax = self.zlimits[n]
1142 self.zmin, self.zmax = self.zlimits[n]
1136
1143
1137 ax.plt = ax.pcolormesh(x, y, z[n].T,
1144 ax.plt = ax.pcolormesh(x, y, z[n].T,
1138 vmin=self.zmin,
1145 vmin=self.zmin,
1139 vmax=self.zmax,
1146 vmax=self.zmax,
1140 cmap=self.cmaps[n]
1147 cmap=self.cmaps[n]
1141 )
1148 )
1142 if self.showprofile:
1149 if self.showprofile:
1143 ax.plot_profile = self.pf_axes[n].plot(data['outlier_rtc'][n], self.y)[0]
1150 ax.plot_profile = self.pf_axes[n].plot(data['outlier_rtc'][n], self.y)[0]
1144 self.pf_axes[n].set_xlabel('')
1151 self.pf_axes[n].set_xlabel('')
1145 else:
1152 else:
1146 if self.zlimits is not None:
1153 if self.zlimits is not None:
1147 self.zmin, self.zmax = self.zlimits[n]
1154 self.zmin, self.zmax = self.zlimits[n]
1148 ax.collections.remove(ax.collections[0])
1155 ax.collections.remove(ax.collections[0])
1149 ax.plt = ax.pcolormesh(x, y, z[n].T ,
1156 ax.plt = ax.pcolormesh(x, y, z[n].T ,
1150 vmin=self.zmin,
1157 vmin=self.zmin,
1151 vmax=self.zmax,
1158 vmax=self.zmax,
1152 cmap=self.cmaps[n]
1159 cmap=self.cmaps[n]
1153 )
1160 )
1154 if self.showprofile:
1161 if self.showprofile:
1155 ax.plot_profile.set_data(data['outlier_rtc'][n], self.y)
1162 ax.plot_profile.set_data(data['outlier_rtc'][n], self.y)
1156 self.pf_axes[n].set_xlabel('')
1163 self.pf_axes[n].set_xlabel('')
1157
1164
1158 class NIncohIntRTIPlot(Plot):
1165 class NIncohIntRTIPlot(Plot):
1159 '''
1166 '''
1160 Plot for data_xxxx object
1167 Plot for data_xxxx object
1161 '''
1168 '''
1162
1169
1163 CODE = 'integrations_rtc' # Range Time Counts
1170 CODE = 'integrations_rtc' # Range Time Counts
1164 colormap = 'BuGn'
1171 colormap = 'BuGn'
1165 plot_type = 'pcolorbuffer'
1172 plot_type = 'pcolorbuffer'
1166
1173
1167 def setup(self):
1174 def setup(self):
1168 self.xaxis = 'time'
1175 self.xaxis = 'time'
1169 self.ncols = 1
1176 self.ncols = 1
1170 self.nrows = self.data.shape('integrations_rtc')[0]
1177 self.nrows = self.data.shape('integrations_rtc')[0]
1171 self.nplots = self.nrows
1178 self.nplots = self.nrows
1172 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
1179 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
1173
1180
1174
1181
1175 if not self.xlabel:
1182 if not self.xlabel:
1176 self.xlabel = 'Time'
1183 self.xlabel = 'Time'
1177
1184
1178 self.ylabel = 'Height [km]'
1185 self.ylabel = 'Height [km]'
1179 if not self.titles:
1186 if not self.titles:
1180 self.titles = ['Integration Ch:{}'.format(x) for x in range(self.nrows)]
1187 self.titles = ['Integration Ch:{}'.format(x) for x in range(self.nrows)]
1181
1188
1182 def update(self, dataOut):
1189 def update(self, dataOut):
1183
1190
1184 data = {}
1191 data = {}
1185 data['integrations_rtc'] = dataOut.nIncohInt
1192 data['integrations_rtc'] = dataOut.nIncohInt
1186
1193
1187 meta = {}
1194 meta = {}
1188
1195
1189 return data, meta
1196 return data, meta
1190
1197
1191 def plot(self):
1198 def plot(self):
1192 # self.data.normalize_heights()
1199 # self.data.normalize_heights()
1193 self.x = self.data.times
1200 self.x = self.data.times
1194 self.y = self.data.yrange
1201 self.y = self.data.yrange
1195 self.z = self.data['integrations_rtc']
1202 self.z = self.data['integrations_rtc']
1196
1203
1197 #self.z = numpy.ma.masked_invalid(self.z)
1204 #self.z = numpy.ma.masked_invalid(self.z)
1198
1205
1199 if self.decimation is None:
1206 if self.decimation is None:
1200 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1207 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1201 else:
1208 else:
1202 x, y, z = self.fill_gaps(*self.decimate())
1209 x, y, z = self.fill_gaps(*self.decimate())
1203
1210
1204 for n, ax in enumerate(self.axes):
1211 for n, ax in enumerate(self.axes):
1205
1212
1206 self.zmax = self.zmax if self.zmax is not None else numpy.max(
1213 self.zmax = self.zmax if self.zmax is not None else numpy.max(
1207 self.z[n])
1214 self.z[n])
1208 self.zmin = self.zmin if self.zmin is not None else numpy.min(
1215 self.zmin = self.zmin if self.zmin is not None else numpy.min(
1209 self.z[n])
1216 self.z[n])
1210 data = self.data[-1]
1217 data = self.data[-1]
1211 if ax.firsttime:
1218 if ax.firsttime:
1212 if self.zlimits is not None:
1219 if self.zlimits is not None:
1213 self.zmin, self.zmax = self.zlimits[n]
1220 self.zmin, self.zmax = self.zlimits[n]
1214
1221
1215 ax.plt = ax.pcolormesh(x, y, z[n].T,
1222 ax.plt = ax.pcolormesh(x, y, z[n].T,
1216 vmin=self.zmin,
1223 vmin=self.zmin,
1217 vmax=self.zmax,
1224 vmax=self.zmax,
1218 cmap=self.cmaps[n]
1225 cmap=self.cmaps[n]
1219 )
1226 )
1220 if self.showprofile:
1227 if self.showprofile:
1221 ax.plot_profile = self.pf_axes[n].plot(data['integrations_rtc'][n], self.y)[0]
1228 ax.plot_profile = self.pf_axes[n].plot(data['integrations_rtc'][n], self.y)[0]
1222 self.pf_axes[n].set_xlabel('')
1229 self.pf_axes[n].set_xlabel('')
1223 else:
1230 else:
1224 if self.zlimits is not None:
1231 if self.zlimits is not None:
1225 self.zmin, self.zmax = self.zlimits[n]
1232 self.zmin, self.zmax = self.zlimits[n]
1226 ax.collections.remove(ax.collections[0])
1233 ax.collections.remove(ax.collections[0])
1227 ax.plt = ax.pcolormesh(x, y, z[n].T ,
1234 ax.plt = ax.pcolormesh(x, y, z[n].T ,
1228 vmin=self.zmin,
1235 vmin=self.zmin,
1229 vmax=self.zmax,
1236 vmax=self.zmax,
1230 cmap=self.cmaps[n]
1237 cmap=self.cmaps[n]
1231 )
1238 )
1232 if self.showprofile:
1239 if self.showprofile:
1233 ax.plot_profile.set_data(data['integrations_rtc'][n], self.y)
1240 ax.plot_profile.set_data(data['integrations_rtc'][n], self.y)
1234 self.pf_axes[n].set_xlabel('')
1241 self.pf_axes[n].set_xlabel('')
@@ -1,698 +1,725
1 ''''
1 ''''
2 Created on Set 9, 2015
2 Created on Set 9, 2015
3
3
4 @author: roj-idl71 Karim Kuyeng
4 @author: roj-idl71 Karim Kuyeng
5
5
6 @update: 2021, Joab Apaza
6 @update: 2021, Joab Apaza
7 '''
7 '''
8
8
9 import os
9 import os
10 import sys
10 import sys
11 import glob
11 import glob
12 import fnmatch
12 import fnmatch
13 import datetime
13 import datetime
14 import time
14 import time
15 import re
15 import re
16 import h5py
16 import h5py
17 import numpy
17 import numpy
18
18
19 try:
19 try:
20 from gevent import sleep
20 from gevent import sleep
21 except:
21 except:
22 from time import sleep
22 from time import sleep
23
23
24 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
24 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader,ProcessingHeader
25 from schainpy.model.data.jrodata import Voltage
25 from schainpy.model.data.jrodata import Voltage
26 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
26 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
27 from numpy import imag
27 from numpy import imag
28 from schainpy.utils import log
28 from schainpy.utils import log
29
29
30
30
31 class AMISRReader(ProcessingUnit):
31 class AMISRReader(ProcessingUnit):
32 '''
32 '''
33 classdocs
33 classdocs
34 '''
34 '''
35
35
36 def __init__(self):
36 def __init__(self):
37 '''
37 '''
38 Constructor
38 Constructor
39 '''
39 '''
40
40
41 ProcessingUnit.__init__(self)
41 ProcessingUnit.__init__(self)
42
42
43 self.set = None
43 self.set = None
44 self.subset = None
44 self.subset = None
45 self.extension_file = '.h5'
45 self.extension_file = '.h5'
46 self.dtc_str = 'dtc'
46 self.dtc_str = 'dtc'
47 self.dtc_id = 0
47 self.dtc_id = 0
48 self.status = True
48 self.status = True
49 self.isConfig = False
49 self.isConfig = False
50 self.dirnameList = []
50 self.dirnameList = []
51 self.filenameList = []
51 self.filenameList = []
52 self.fileIndex = None
52 self.fileIndex = None
53 self.flagNoMoreFiles = False
53 self.flagNoMoreFiles = False
54 self.flagIsNewFile = 0
54 self.flagIsNewFile = 0
55 self.filename = ''
55 self.filename = ''
56 self.amisrFilePointer = None
56 self.amisrFilePointer = None
57
57
58 self.beamCodeMap = None
58 self.beamCodeMap = None
59 self.azimuthList = []
59 self.azimuthList = []
60 self.elevationList = []
60 self.elevationList = []
61 self.dataShape = None
61 self.dataShape = None
62 self.flag_old_beams = False
62 self.flag_old_beams = False
63
63
64
64
65 self.profileIndex = 0
65 self.profileIndex = 0
66
66
67
67
68 self.beamCodeByFrame = None
68 self.beamCodeByFrame = None
69 self.radacTimeByFrame = None
69 self.radacTimeByFrame = None
70
70
71 self.dataset = None
71 self.dataset = None
72
72
73 self.__firstFile = True
73 self.__firstFile = True
74
74
75 self.buffer = None
75 self.buffer = None
76
76
77 self.timezone = 'ut'
77 self.timezone = 'ut'
78
78
79 self.__waitForNewFile = 20
79 self.__waitForNewFile = 20
80 self.__filename_online = None
80 self.__filename_online = None
81 #Is really necessary create the output object in the initializer
81 #Is really necessary create the output object in the initializer
82 self.dataOut = Voltage()
82 self.dataOut = Voltage()
83 self.dataOut.error=False
83 self.dataOut.error=False
84 self.margin_days = 1
84 self.margin_days = 1
85
85
86 def setup(self,path=None,
86 def setup(self,path=None,
87 startDate=None,
87 startDate=None,
88 endDate=None,
88 endDate=None,
89 startTime=None,
89 startTime=None,
90 endTime=None,
90 endTime=None,
91 walk=True,
91 walk=True,
92 timezone='ut',
92 timezone='ut',
93 all=0,
93 all=0,
94 code = None,
94 code = None,
95 nCode = 1,
95 nCode = 1,
96 nBaud = 0,
96 nBaud = 0,
97 nOsamp = None,
97 online=False,
98 online=False,
98 old_beams=False,
99 old_beams=False,
99 margin_days=1,
100 margin_days=1,
100 nFFT = 1,
101 nFFT = None,
101 nChannels = None,
102 nChannels = None,
102 ):
103 ):
103
104
104
105
105
106
106 self.timezone = timezone
107 self.timezone = timezone
107 self.all = all
108 self.all = all
108 self.online = online
109 self.online = online
109 self.flag_old_beams = old_beams
110 self.flag_old_beams = old_beams
110 self.code = code
111 self.code = code
111 self.nCode = int(nCode)
112 self.nCode = int(nCode)
112 self.nBaud = int(nBaud)
113 self.nBaud = int(nBaud)
114 self.nOsamp = int(nOsamp)
113 self.margin_days = margin_days
115 self.margin_days = margin_days
114 self.__sampleRate = None
116 self.__sampleRate = None
115
117
116 self.nFFT = nFFT
118 self.nFFT = nFFT
117 self.nChannels = nChannels
119 self.nChannels = nChannels
118
120
119 #self.findFiles()
121 #self.findFiles()
120 if not(online):
122 if not(online):
121 #Busqueda de archivos offline
123 #Busqueda de archivos offline
122 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
124 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
123 else:
125 else:
124 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
126 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
125
127
126 if not(self.filenameList):
128 if not(self.filenameList):
127 raise schainpy.admin.SchainWarning("There is no files into the folder: %s"%(path))
129 raise schainpy.admin.SchainWarning("There is no files into the folder: %s"%(path))
128 #sys.exit(0)
130 #sys.exit(0)
129 self.dataOut.error = True
131 self.dataOut.error = True
130
132
131 self.fileIndex = 0
133 self.fileIndex = 0
132
134
133 self.readNextFile(online)
135 self.readNextFile(online)
134
136
135 '''
137 '''
136 Add code
138 Add code
137 '''
139 '''
138 self.isConfig = True
140 self.isConfig = True
139 # print("Setup Done")
141 # print("Setup Done")
140 pass
142 pass
141
143
142
144
143 def readAMISRHeader(self,fp):
145 def readAMISRHeader(self,fp):
144
146
145 if self.isConfig and (not self.flagNoMoreFiles):
147 if self.isConfig and (not self.flagNoMoreFiles):
146 newShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
148 newShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
147 if self.dataShape != newShape and newShape != None:
149 if self.dataShape != newShape and newShape != None:
148 raise schainpy.admin.SchainError("NEW FILE HAS A DIFFERENT SHAPE: ")
150 raise schainpy.admin.SchainError("NEW FILE HAS A DIFFERENT SHAPE: ")
149 print(self.dataShape,newShape,"\n")
151 print(self.dataShape,newShape,"\n")
150 return 0
152 return 0
151 else:
153 else:
152 self.dataShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
154 self.dataShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
153
155
154
156
155 header = 'Raw11/Data/RadacHeader'
157 header = 'Raw11/Data/RadacHeader'
156 if self.nChannels == None:
158 if self.nChannels == None:
157 expFile = fp['Setup/ExperimentFile'][()].decode()
159 expFile = fp['Setup/Experimentfile'][()].decode()
158 linesExp = expFile.split("\n")
160 linesExp = expFile.split("\n")
159 a = [line for line in linesExp if "nbeamcodes" in line]
161 a = [line for line in linesExp if "nbeamcodes" in line]
160 self.nChannels = int(a[0][11:])
162 self.nChannels = int(a[0][11:])
161
163
162 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
164 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
163 if (self.startDate > datetime.date(2021, 7, 15)) or self.flag_old_beams: #Se cambió la forma de extracción de Apuntes el 17 o forzar con flag de reorganización
165 if (self.startDate > datetime.date(2021, 7, 15)) or self.flag_old_beams: #Se cambió la forma de extracción de Apuntes el 17 o forzar con flag de reorganización
164 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
166 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
165 self.trueBeams = self.beamcodeFile.split("\n")
167 self.trueBeams = self.beamcodeFile.split("\n")
166 self.trueBeams.pop()#remove last
168 self.trueBeams.pop()#remove last
169 if self.nFFT == None:
170 log.error("FFT or number of repetitions per channels is needed",self.name)
167 beams_idx = [k*self.nFFT for k in range(self.nChannels)]
171 beams_idx = [k*self.nFFT for k in range(self.nChannels)]
168 beams = [self.trueBeams[b] for b in beams_idx]
172 beams = [self.trueBeams[b] for b in beams_idx]
169 self.beamCode = [int(x, 16) for x in beams]
173 self.beamCode = [int(x, 16) for x in beams]
170
174
171 else:
175 else:
172 _beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
176 _beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
173 self.beamCode = _beamCode[0,:]
177 self.beamCode = _beamCode[0,:]
174
178
175 if self.beamCodeMap == None:
179 if self.beamCodeMap == None:
176 self.beamCodeMap = fp['Setup/BeamcodeMap']
180 self.beamCodeMap = fp['Setup/BeamcodeMap']
177 for beam in self.beamCode:
181 for beam in self.beamCode:
178 beamAziElev = numpy.where(self.beamCodeMap[:,0]==beam)
182 beamAziElev = numpy.where(self.beamCodeMap[:,0]==beam)
179 beamAziElev = beamAziElev[0].squeeze()
183 beamAziElev = beamAziElev[0].squeeze()
180 self.azimuthList.append(self.beamCodeMap[beamAziElev,1])
184 self.azimuthList.append(self.beamCodeMap[beamAziElev,1])
181 self.elevationList.append(self.beamCodeMap[beamAziElev,2])
185 self.elevationList.append(self.beamCodeMap[beamAziElev,2])
182 #print("Beamssss: ",self.beamCodeMap[beamAziElev,1],self.beamCodeMap[beamAziElev,2])
186 #print("Beamssss: ",self.beamCodeMap[beamAziElev,1],self.beamCodeMap[beamAziElev,2])
183 #print(self.beamCode)
187 #print(self.beamCode)
184 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
188 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
185 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
189 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
186 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
190 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
187 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
191 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
188 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
192 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
189 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
193 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
190 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
194 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
191 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
195 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
192 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
196 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
193 self.frequency = fp.get('Rx/Frequency')
197 self.frequency = fp.get('Rx/Frequency')
194 txAus = fp.get('Raw11/Data/Pulsewidth')
198 txAus = fp.get('Raw11/Data/Pulsewidth') #seconds
195 self.baud = fp.get('Raw11/Data/TxBaud')
199 self.baud = fp.get('Raw11/Data/TxBaud')
196 sampleRate = fp.get('Rx/SampleRate')
200 sampleRate = fp.get('Rx/SampleRate')
197 self.__sampleRate = sampleRate[()]
201 self.__sampleRate = sampleRate[()]
198 self.nblocks = self.pulseCount.shape[0] #nblocks
202 self.nblocks = self.pulseCount.shape[0] #nblocks
199 self.profPerBlockRAW = self.pulseCount.shape[1] #profiles per block in raw data
203 self.profPerBlockRAW = self.pulseCount.shape[1] #profiles per block in raw data
200 self.nprofiles = self.pulseCount.shape[1] #nprofile
204 self.nprofiles = self.pulseCount.shape[1] #nprofile
201 self.nsa = self.nsamplesPulse[0,0] #ngates
205 self.nsa = self.nsamplesPulse[0,0] #ngates
202 self.nchannels = len(self.beamCode)
206 self.nchannels = len(self.beamCode)
203 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
207 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
204 #print("IPPS secs: ",self.ippSeconds)
208 #print("IPPS secs: ",self.ippSeconds)
205 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
209 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
206 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
210 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
207
211
208 #filling radar controller header parameters
212 #filling radar controller header parameters
209 self.__ippKm = self.ippSeconds *.15*1e6 # in km
213 self.__ippKm = self.ippSeconds *.15*1e6 # in km
210 self.__txA = (txAus[()])*.15 #(ipp[us]*.15km/1us) in km
214 #self.__txA = txAus[()]*.15 #(ipp[us]*.15km/1us) in km
215 self.__txA = txAus[()] #seconds
216 self.__txAKm = self.__txA*1e6*.15
211 self.__txB = 0
217 self.__txB = 0
212 nWindows=1
218 nWindows=1
213 self.__nSamples = self.nsa
219 self.__nSamples = self.nsa
214 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
220 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
215 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
221 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
216 #print("amisr-ipp:",self.ippSeconds, self.__ippKm)
222 #print("amisr-ipp:",self.ippSeconds, self.__ippKm)
217 #for now until understand why the code saved is different (code included even though code not in tuf file)
223 #for now until understand why the code saved is different (code included even though code not in tuf file)
218 #self.__codeType = 0
224 #self.__codeType = 0
219 # self.__nCode = None
225 # self.__nCode = None
220 # self.__nBaud = None
226 # self.__nBaud = None
221 self.__code = self.code
227 self.__code = self.code
222 self.__codeType = 0
228 self.__codeType = 0
223 if self.code != None:
229 if self.code != None:
224 self.__codeType = 1
230 self.__codeType = 1
225 self.__nCode = self.nCode
231 self.__nCode = self.nCode
226 self.__nBaud = self.nBaud
232 self.__nBaud = self.nBaud
233 self.__baudTX = self.__txA/(self.nBaud)
227 #self.__code = 0
234 #self.__code = 0
228
235
229 #filling system header parameters
236 #filling system header parameters
230 self.__nSamples = self.nsa
237 self.__nSamples = self.nsa
231 self.newProfiles = self.nprofiles/self.nchannels
238 self.newProfiles = self.nprofiles/self.nchannels
232 self.__channelList = [n for n in range(self.nchannels)]
239 self.__channelList = [n for n in range(self.nchannels)]
233
240
234 self.__frequency = self.frequency[0][0]
241 self.__frequency = self.frequency[0][0]
235
242
236
243
237 return 1
244 return 1
238
245
239
246
240 def createBuffers(self):
247 def createBuffers(self):
241
248
242 pass
249 pass
243
250
244 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
251 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
245 self.path = path
252 self.path = path
246 self.startDate = startDate
253 self.startDate = startDate
247 self.endDate = endDate
254 self.endDate = endDate
248 self.startTime = startTime
255 self.startTime = startTime
249 self.endTime = endTime
256 self.endTime = endTime
250 self.walk = walk
257 self.walk = walk
251
258
252 def __checkPath(self):
259 def __checkPath(self):
253 if os.path.exists(self.path):
260 if os.path.exists(self.path):
254 self.status = 1
261 self.status = 1
255 else:
262 else:
256 self.status = 0
263 self.status = 0
257 print('Path:%s does not exists'%self.path)
264 print('Path:%s does not exists'%self.path)
258
265
259 return
266 return
260
267
261
268
262 def __selDates(self, amisr_dirname_format):
269 def __selDates(self, amisr_dirname_format):
263 try:
270 try:
264 year = int(amisr_dirname_format[0:4])
271 year = int(amisr_dirname_format[0:4])
265 month = int(amisr_dirname_format[4:6])
272 month = int(amisr_dirname_format[4:6])
266 dom = int(amisr_dirname_format[6:8])
273 dom = int(amisr_dirname_format[6:8])
267 thisDate = datetime.date(year,month,dom)
274 thisDate = datetime.date(year,month,dom)
268 #margen de un día extra, igual luego se filtra for fecha y hora
275 #margen de un día extra, igual luego se filtra for fecha y hora
269 if (thisDate>=(self.startDate - datetime.timedelta(days=self.margin_days)) and thisDate <= (self.endDate)+ datetime.timedelta(days=1)):
276 if (thisDate>=(self.startDate - datetime.timedelta(days=self.margin_days)) and thisDate <= (self.endDate)+ datetime.timedelta(days=1)):
270 return amisr_dirname_format
277 return amisr_dirname_format
271 except:
278 except:
272 return None
279 return None
273
280
274
281
275 def __findDataForDates(self,online=False):
282 def __findDataForDates(self,online=False):
276
283
277 if not(self.status):
284 if not(self.status):
278 return None
285 return None
279
286
280 pat = '\d+.\d+'
287 pat = '\d+.\d+'
281 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
288 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
282 dirnameList = [x for x in dirnameList if x!=None]
289 dirnameList = [x for x in dirnameList if x!=None]
283 dirnameList = [x.string for x in dirnameList]
290 dirnameList = [x.string for x in dirnameList]
284 if not(online):
291 if not(online):
285 dirnameList = [self.__selDates(x) for x in dirnameList]
292 dirnameList = [self.__selDates(x) for x in dirnameList]
286 dirnameList = [x for x in dirnameList if x!=None]
293 dirnameList = [x for x in dirnameList if x!=None]
287 if len(dirnameList)>0:
294 if len(dirnameList)>0:
288 self.status = 1
295 self.status = 1
289 self.dirnameList = dirnameList
296 self.dirnameList = dirnameList
290 self.dirnameList.sort()
297 self.dirnameList.sort()
291 else:
298 else:
292 self.status = 0
299 self.status = 0
293 return None
300 return None
294
301
295 def __getTimeFromData(self):
302 def __getTimeFromData(self):
296 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
303 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
297 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
304 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
298
305
299 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
306 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
300 print('........................................')
307 print('........................................')
301 filter_filenameList = []
308 filter_filenameList = []
302 self.filenameList.sort()
309 self.filenameList.sort()
303 total_files = len(self.filenameList)
310 total_files = len(self.filenameList)
304 #for i in range(len(self.filenameList)-1):
311 #for i in range(len(self.filenameList)-1):
305 for i in range(total_files):
312 for i in range(total_files):
306 filename = self.filenameList[i]
313 filename = self.filenameList[i]
307 #print("file-> ",filename)
314 #print("file-> ",filename)
308 try:
315 try:
309 fp = h5py.File(filename,'r')
316 fp = h5py.File(filename,'r')
310 time_str = fp.get('Time/RadacTimeString')
317 time_str = fp.get('Time/RadacTimeString')
311
318
312 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
319 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
313 #startDateTimeStr_File = "2019-12-16 09:21:11"
320 #startDateTimeStr_File = "2019-12-16 09:21:11"
314 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
321 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
315 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
322 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
316
323
317 #endDateTimeStr_File = "2019-12-16 11:10:11"
324 #endDateTimeStr_File = "2019-12-16 11:10:11"
318 endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
325 endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
319 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
326 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
320 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
327 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
321
328
322 fp.close()
329 fp.close()
323
330
324 #print("check time", startDateTime_File)
331 #print("check time", startDateTime_File)
325 if self.timezone == 'lt':
332 if self.timezone == 'lt':
326 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
333 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
327 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
334 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
328 if (startDateTime_File >=startDateTime_Reader and endDateTime_File<=endDateTime_Reader):
335 if (startDateTime_File >=startDateTime_Reader and endDateTime_File<=endDateTime_Reader):
329 filter_filenameList.append(filename)
336 filter_filenameList.append(filename)
330
337
331 if (startDateTime_File>endDateTime_Reader):
338 if (startDateTime_File>endDateTime_Reader):
332 break
339 break
333 except Exception as e:
340 except Exception as e:
334 log.warning("Error opening file {} -> {}".format(os.path.split(filename)[1],e))
341 log.warning("Error opening file {} -> {}".format(os.path.split(filename)[1],e))
335
342
336 filter_filenameList.sort()
343 filter_filenameList.sort()
337 self.filenameList = filter_filenameList
344 self.filenameList = filter_filenameList
338
345
339 return 1
346 return 1
340
347
341 def __filterByGlob1(self, dirName):
348 def __filterByGlob1(self, dirName):
342 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
349 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
343 filter_files.sort()
350 filter_files.sort()
344 filterDict = {}
351 filterDict = {}
345 filterDict.setdefault(dirName)
352 filterDict.setdefault(dirName)
346 filterDict[dirName] = filter_files
353 filterDict[dirName] = filter_files
347 return filterDict
354 return filterDict
348
355
349 def __getFilenameList(self, fileListInKeys, dirList):
356 def __getFilenameList(self, fileListInKeys, dirList):
350 for value in fileListInKeys:
357 for value in fileListInKeys:
351 dirName = list(value.keys())[0]
358 dirName = list(value.keys())[0]
352 for file in value[dirName]:
359 for file in value[dirName]:
353 filename = os.path.join(dirName, file)
360 filename = os.path.join(dirName, file)
354 self.filenameList.append(filename)
361 self.filenameList.append(filename)
355
362
356
363
357 def __selectDataForTimes(self, online=False):
364 def __selectDataForTimes(self, online=False):
358 #aun no esta implementado el filtro for tiempo
365 #aun no esta implementado el filtro for tiempo
359 if not(self.status):
366 if not(self.status):
360 return None
367 return None
361
368
362 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
369 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
363 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
370 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
364 self.__getFilenameList(fileListInKeys, dirList)
371 self.__getFilenameList(fileListInKeys, dirList)
365 if not(online):
372 if not(online):
366 #filtro por tiempo
373 #filtro por tiempo
367 if not(self.all):
374 if not(self.all):
368 self.__getTimeFromData()
375 self.__getTimeFromData()
369
376
370 if len(self.filenameList)>0:
377 if len(self.filenameList)>0:
371 self.status = 1
378 self.status = 1
372 self.filenameList.sort()
379 self.filenameList.sort()
373 else:
380 else:
374 self.status = 0
381 self.status = 0
375 return None
382 return None
376
383
377 else:
384 else:
378 #get the last file - 1
385 #get the last file - 1
379 self.filenameList = [self.filenameList[-2]]
386 self.filenameList = [self.filenameList[-2]]
380 new_dirnameList = []
387 new_dirnameList = []
381 for dirname in self.dirnameList:
388 for dirname in self.dirnameList:
382 junk = numpy.array([dirname in x for x in self.filenameList])
389 junk = numpy.array([dirname in x for x in self.filenameList])
383 junk_sum = junk.sum()
390 junk_sum = junk.sum()
384 if junk_sum > 0:
391 if junk_sum > 0:
385 new_dirnameList.append(dirname)
392 new_dirnameList.append(dirname)
386 self.dirnameList = new_dirnameList
393 self.dirnameList = new_dirnameList
387 return 1
394 return 1
388
395
389 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
396 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
390 endTime=datetime.time(23,59,59),walk=True):
397 endTime=datetime.time(23,59,59),walk=True):
391
398
392 if endDate ==None:
399 if endDate ==None:
393 startDate = datetime.datetime.utcnow().date()
400 startDate = datetime.datetime.utcnow().date()
394 endDate = datetime.datetime.utcnow().date()
401 endDate = datetime.datetime.utcnow().date()
395
402
396 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
403 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
397
404
398 self.__checkPath()
405 self.__checkPath()
399
406
400 self.__findDataForDates(online=True)
407 self.__findDataForDates(online=True)
401
408
402 self.dirnameList = [self.dirnameList[-1]]
409 self.dirnameList = [self.dirnameList[-1]]
403
410
404 self.__selectDataForTimes(online=True)
411 self.__selectDataForTimes(online=True)
405
412
406 return
413 return
407
414
408
415
409 def searchFilesOffLine(self,
416 def searchFilesOffLine(self,
410 path,
417 path,
411 startDate,
418 startDate,
412 endDate,
419 endDate,
413 startTime=datetime.time(0,0,0),
420 startTime=datetime.time(0,0,0),
414 endTime=datetime.time(23,59,59),
421 endTime=datetime.time(23,59,59),
415 walk=True):
422 walk=True):
416
423
417 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
424 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
418
425
419 self.__checkPath()
426 self.__checkPath()
420
427
421 self.__findDataForDates()
428 self.__findDataForDates()
422
429
423 self.__selectDataForTimes()
430 self.__selectDataForTimes()
424
431
425 for i in range(len(self.filenameList)):
432 for i in range(len(self.filenameList)):
426 print("%s" %(self.filenameList[i]))
433 print("%s" %(self.filenameList[i]))
427
434
428 return
435 return
429
436
430 def __setNextFileOffline(self):
437 def __setNextFileOffline(self):
431
438
432 try:
439 try:
433 self.filename = self.filenameList[self.fileIndex]
440 self.filename = self.filenameList[self.fileIndex]
434 self.amisrFilePointer = h5py.File(self.filename,'r')
441 self.amisrFilePointer = h5py.File(self.filename,'r')
435 self.fileIndex += 1
442 self.fileIndex += 1
436 except:
443 except:
437 self.flagNoMoreFiles = 1
444 self.flagNoMoreFiles = 1
438 raise schainpy.admin.SchainError('No more files to read')
445 raise schainpy.admin.SchainError('No more files to read')
439 return 0
446 return 0
440
447
441 self.flagIsNewFile = 1
448 self.flagIsNewFile = 1
442 print("Setting the file: %s"%self.filename)
449 print("Setting the file: %s"%self.filename)
443
450
444 return 1
451 return 1
445
452
446
453
447 def __setNextFileOnline(self):
454 def __setNextFileOnline(self):
448 filename = self.filenameList[0]
455 filename = self.filenameList[0]
449 if self.__filename_online != None:
456 if self.__filename_online != None:
450 self.__selectDataForTimes(online=True)
457 self.__selectDataForTimes(online=True)
451 filename = self.filenameList[0]
458 filename = self.filenameList[0]
452 wait = 0
459 wait = 0
453 self.__waitForNewFile=300 ## DEBUG:
460 self.__waitForNewFile=300 ## DEBUG:
454 while self.__filename_online == filename:
461 while self.__filename_online == filename:
455 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
462 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
456 if wait == 5:
463 if wait == 5:
457 self.flagNoMoreFiles = 1
464 self.flagNoMoreFiles = 1
458 return 0
465 return 0
459 sleep(self.__waitForNewFile)
466 sleep(self.__waitForNewFile)
460 self.__selectDataForTimes(online=True)
467 self.__selectDataForTimes(online=True)
461 filename = self.filenameList[0]
468 filename = self.filenameList[0]
462 wait += 1
469 wait += 1
463
470
464 self.__filename_online = filename
471 self.__filename_online = filename
465
472
466 self.amisrFilePointer = h5py.File(filename,'r')
473 self.amisrFilePointer = h5py.File(filename,'r')
467 self.flagIsNewFile = 1
474 self.flagIsNewFile = 1
468 self.filename = filename
475 self.filename = filename
469 print("Setting the file: %s"%self.filename)
476 print("Setting the file: %s"%self.filename)
470 return 1
477 return 1
471
478
472
479
473 def readData(self):
480 def readData(self):
474 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
481 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
475 re = buffer[:,:,:,0]
482 re = buffer[:,:,:,0]
476 im = buffer[:,:,:,1]
483 im = buffer[:,:,:,1]
477 dataset = re + im*1j
484 dataset = re + im*1j
478
485
479 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
486 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
480 timeset = self.radacTime[:,0]
487 timeset = self.radacTime[:,0]
481
488
482 return dataset,timeset
489 return dataset,timeset
483
490
484 def reshapeData(self):
491 def reshapeData(self):
485 #print(self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa)
492 #print(self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa)
486 channels = self.beamCodeByPulse[0,:]
493 channels = self.beamCodeByPulse[0,:]
487 nchan = self.nchannels
494 nchan = self.nchannels
488 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
495 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
489 nblocks = self.nblocks
496 nblocks = self.nblocks
490 nsamples = self.nsa
497 nsamples = self.nsa
491 #print("Channels: ",self.nChannels)
498 #print("Channels: ",self.nChannels)
492 #Dimensions : nChannels, nProfiles, nSamples
499 #Dimensions : nChannels, nProfiles, nSamples
493 new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
500 new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
494 ############################################
501 ############################################
495 profPerCH = int(self.profPerBlockRAW / (self.nFFT* self.nChannels))
502 profPerCH = int(self.profPerBlockRAW / (self.nFFT* self.nChannels))
496
503 #profPerCH = int(self.profPerBlockRAW / self.nChannels)
497 for thisChannel in range(nchan):
504 for thisChannel in range(nchan):
498
505
499 idx_ch = [thisChannel+(k*nchan*self.nFFT) for k in range(profPerCH)]
506
507 idx_ch = [self.nFFT*(thisChannel + nchan*k) for k in range(profPerCH)]
508 #print(idx_ch)
500 if self.nFFT > 1:
509 if self.nFFT > 1:
501 aux = [numpy.arange(i, i+self.nFFT) for i in idx_ch]
510 aux = [numpy.arange(i, i+self.nFFT) for i in idx_ch]
502 idx_ch = None
511 idx_ch = None
503 idx_ch =aux
512 idx_ch =aux
504 idx_ch = numpy.array(idx_ch, dtype=int).flatten()
513 idx_ch = numpy.array(idx_ch, dtype=int).flatten()
505 else:
514 else:
506 idx_ch = numpy.array(idx_ch, dtype=int)
515 idx_ch = numpy.array(idx_ch, dtype=int)
507
516
508 #print(thisChannel,profPerCH,idx_ch)
517 #print(thisChannel,profPerCH,idx_ch)
509 #print(numpy.where(channels==self.beamCode[thisChannel])[0])
518 #print(numpy.where(channels==self.beamCode[thisChannel])[0])
510 #new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[thisChannel])[0],:]
519 #new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[thisChannel])[0],:]
511 new_block[:,thisChannel,:,:] = self.dataset[:,idx_ch,:]
520 new_block[:,thisChannel,:,:] = self.dataset[:,idx_ch,:]
512
521
513 new_block = numpy.transpose(new_block, (1,0,2,3))
522 new_block = numpy.transpose(new_block, (1,0,2,3))
514 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
523 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
515
524
516 return new_block
525 return new_block
517
526
518 def updateIndexes(self):
527 def updateIndexes(self):
519
528
520 pass
529 pass
521
530
522 def fillJROHeader(self):
531 def fillJROHeader(self):
523
532
524 #fill radar controller header
533 #fill radar controller header
525 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
534
526 txA=self.__txA,
527 txB=0,
528 nWindows=1,
529 nHeights=self.__nSamples,
530 firstHeight=self.__firstHeight,
531 deltaHeight=self.__deltaHeight,
532 codeType=self.__codeType,
533 nCode=self.__nCode, nBaud=self.__nBaud,
534 code = self.__code,
535 fClock=self.__sampleRate)
536 #fill system header
535 #fill system header
537 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
536 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
538 nProfiles=self.newProfiles,
537 nProfiles=self.newProfiles,
539 nChannels=len(self.__channelList),
538 nChannels=len(self.__channelList),
540 adcResolution=14,
539 adcResolution=14,
541 pciDioBusWidth=32)
540 pciDioBusWidth=32)
542
541
543 self.dataOut.type = "Voltage"
542 self.dataOut.type = "Voltage"
544 self.dataOut.data = None
543 self.dataOut.data = None
545 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
544 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
546 # self.dataOut.nChannels = 0
545 # self.dataOut.nChannels = 0
547
546
548 # self.dataOut.nHeights = 0
547 # self.dataOut.nHeights = 0
549
548
550 self.dataOut.nProfiles = self.newProfiles*self.nblocks
549 self.dataOut.nProfiles = self.newProfiles*self.nblocks
551 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
550 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
552 ranges = numpy.reshape(self.rangeFromFile[()],(-1))
551 ranges = numpy.reshape(self.rangeFromFile[()],(-1))
553 self.dataOut.heightList = ranges/1000.0 #km
552 self.dataOut.heightList = ranges/1000.0 #km
554 self.dataOut.channelList = self.__channelList
553 self.dataOut.channelList = self.__channelList
554
555 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
555 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
556
556
557 # self.dataOut.channelIndexList = None
557 # self.dataOut.channelIndexList = None
558
558
559
559
560 self.dataOut.azimuthList = numpy.array(self.azimuthList)
560 self.dataOut.azimuthList = numpy.array(self.azimuthList)
561 self.dataOut.elevationList = numpy.array(self.elevationList)
561 self.dataOut.elevationList = numpy.array(self.elevationList)
562 self.dataOut.codeList = numpy.array(self.beamCode)
562 self.dataOut.codeList = numpy.array(self.beamCode)
563
564
565
566
563 #print(self.dataOut.elevationList)
567 #print(self.dataOut.elevationList)
564 self.dataOut.flagNoData = True
568 self.dataOut.flagNoData = True
565
569
566 #Set to TRUE if the data is discontinuous
570 #Set to TRUE if the data is discontinuous
567 self.dataOut.flagDiscontinuousBlock = False
571 self.dataOut.flagDiscontinuousBlock = False
568
572
569 self.dataOut.utctime = None
573 self.dataOut.utctime = None
570
574
571 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
575 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
572 if self.timezone == 'lt':
576 if self.timezone == 'lt':
573 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
577 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
574 else:
578 else:
575 self.dataOut.timeZone = 0 #by default time is UTC
579 self.dataOut.timeZone = 0 #by default time is UTC
576
580
577 self.dataOut.dstFlag = 0
581 self.dataOut.dstFlag = 0
578 self.dataOut.errorCount = 0
582 self.dataOut.errorCount = 0
579 self.dataOut.nCohInt = 1
583 self.dataOut.nCohInt = 1
580 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
584 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
581 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
585 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
582 self.dataOut.flagShiftFFT = False
586 self.dataOut.flagShiftFFT = False
583 self.dataOut.ippSeconds = self.ippSeconds
587 self.dataOut.ippSeconds = self.ippSeconds
584 self.dataOut.radar_ipp = self.ippSeconds
588 self.dataOut.ipp = self.__ippKm
585 self.dataOut.pulseLength_TxA = self.__txA/0.15
589
586 self.dataOut.deltaHeight = self.__deltaHeight
587 #Time interval between profiles
588 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
589
590
590 self.dataOut.frequency = self.__frequency
591 self.dataOut.frequency = self.__frequency
591 self.dataOut.realtime = self.online
592 self.dataOut.realtime = self.online
593
594 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
595 txA=self.__txAKm,
596 txB=0,
597 nWindows=1,
598 nHeights=self.__nSamples,
599 firstHeight=self.__firstHeight,
600 codeType=self.__codeType,
601 nCode=self.__nCode, nBaud=self.__nBaud,
602 code = self.__code,
603 nOsamp=self.nOsamp,
604 frequency = self.__frequency,
605 sampleRate= self.__sampleRate,
606 fClock=self.__sampleRate)
607
608
609 self.dataOut.radarControllerHeaderObj.heightList = ranges/1000.0 #km
610 self.dataOut.radarControllerHeaderObj.heightResolution = self.__deltaHeight
611 self.dataOut.radarControllerHeaderObj.rangeIpp = self.__ippKm #km
612 self.dataOut.radarControllerHeaderObj.rangeTxA = self.__txA*1e6*.15 #km
613 self.dataOut.radarControllerHeaderObj.nChannels = self.nchannels
614 self.dataOut.radarControllerHeaderObj.channelList = self.__channelList
615 self.dataOut.radarControllerHeaderObj.azimuthList = self.azimuthList
616 self.dataOut.radarControllerHeaderObj.elevationList = self.elevationList
617 self.dataOut.radarControllerHeaderObj.dtype = "Voltage"
618 self.dataOut.ippSeconds = self.ippSeconds
592 pass
619 pass
593
620
594 def readNextFile(self,online=False):
621 def readNextFile(self,online=False):
595
622
596 if not(online):
623 if not(online):
597 newFile = self.__setNextFileOffline()
624 newFile = self.__setNextFileOffline()
598 else:
625 else:
599 newFile = self.__setNextFileOnline()
626 newFile = self.__setNextFileOnline()
600
627
601 if not(newFile):
628 if not(newFile):
602 self.dataOut.error = True
629 self.dataOut.error = True
603 return 0
630 return 0
604
631
605 if not self.readAMISRHeader(self.amisrFilePointer):
632 if not self.readAMISRHeader(self.amisrFilePointer):
606 self.dataOut.error = True
633 self.dataOut.error = True
607 return 0
634 return 0
608
635
609 self.createBuffers()
636 self.createBuffers()
610 self.fillJROHeader()
637 self.fillJROHeader()
611
638
612 #self.__firstFile = False
639 #self.__firstFile = False
613
640
614
641
615
642
616 self.dataset,self.timeset = self.readData()
643 self.dataset,self.timeset = self.readData()
617
644
618 if self.endDate!=None:
645 if self.endDate!=None:
619 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
646 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
620 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
647 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
621 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
648 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
622 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
649 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
623 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
650 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
624 if self.timezone == 'lt':
651 if self.timezone == 'lt':
625 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
652 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
626 if (startDateTime_File>endDateTime_Reader):
653 if (startDateTime_File>endDateTime_Reader):
627 return 0
654 return 0
628
655
629 self.jrodataset = self.reshapeData()
656 self.jrodataset = self.reshapeData()
630 #----self.updateIndexes()
657 #----self.updateIndexes()
631 self.profileIndex = 0
658 self.profileIndex = 0
632
659
633 return 1
660 return 1
634
661
635
662
636 def __hasNotDataInBuffer(self):
663 def __hasNotDataInBuffer(self):
637 if self.profileIndex >= (self.newProfiles*self.nblocks):
664 if self.profileIndex >= (self.newProfiles*self.nblocks):
638 return 1
665 return 1
639 return 0
666 return 0
640
667
641
668
642 def getData(self):
669 def getData(self):
643
670
644 if self.flagNoMoreFiles:
671 if self.flagNoMoreFiles:
645 self.dataOut.flagNoData = True
672 self.dataOut.flagNoData = True
646 return 0
673 return 0
647
674
648 if self.profileIndex >= (self.newProfiles*self.nblocks): #
675 if self.profileIndex >= (self.newProfiles*self.nblocks): #
649 #if self.__hasNotDataInBuffer():
676 #if self.__hasNotDataInBuffer():
650 if not (self.readNextFile(self.online)):
677 if not (self.readNextFile(self.online)):
651 return 0
678 return 0
652
679
653
680
654 if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
681 if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
655 self.dataOut.flagNoData = True
682 self.dataOut.flagNoData = True
656 return 0
683 return 0
657
684
658 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
685 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
659
686
660 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
687 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
661
688
662 #print("R_t",self.timeset)
689 #print("R_t",self.timeset)
663
690
664 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
691 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
665 #verificar basic header de jro data y ver si es compatible con este valor
692 #verificar basic header de jro data y ver si es compatible con este valor
666 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
693 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
667 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
694 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
668 indexblock = self.profileIndex/self.newProfiles
695 indexblock = self.profileIndex/self.newProfiles
669 #print (indexblock, indexprof)
696 #print (indexblock, indexprof)
670 diffUTC = 0
697 diffUTC = 0
671 t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
698 t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
672
699
673 #print("utc :",indexblock," __ ",t_comp)
700 #print("utc :",indexblock," __ ",t_comp)
674 #print(numpy.shape(self.timeset))
701 #print(numpy.shape(self.timeset))
675 self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
702 self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
676 #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
703 #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
677
704
678 self.dataOut.profileIndex = self.profileIndex
705 self.dataOut.profileIndex = self.profileIndex
679 #print("N profile:",self.profileIndex,self.newProfiles,self.nblocks,self.dataOut.utctime)
706 #print("N profile:",self.profileIndex,self.newProfiles,self.nblocks,self.dataOut.utctime)
680 self.dataOut.flagNoData = False
707 self.dataOut.flagNoData = False
681 # if indexprof == 0:
708 # if indexprof == 0:
682 # print("kamisr: ",self.dataOut.utctime)
709 # print("kamisr: ",self.dataOut.utctime)
683
710
684 self.profileIndex += 1
711 self.profileIndex += 1
685
712
686 return self.dataOut.data #retorno necesario??
713 return self.dataOut.data #retorno necesario??
687
714
688
715
689 def run(self, **kwargs):
716 def run(self, **kwargs):
690 '''
717 '''
691 This method will be called many times so here you should put all your code
718 This method will be called many times so here you should put all your code
692 '''
719 '''
693 #print("running kamisr")
720 #print("running kamisr")
694 if not self.isConfig:
721 if not self.isConfig:
695 self.setup(**kwargs)
722 self.setup(**kwargs)
696 self.isConfig = True
723 self.isConfig = True
697
724
698 self.getData()
725 self.getData()
@@ -1,687 +1,809
1 import os
1 import os
2 import time
2 import time
3 import datetime
3 import datetime
4
4
5 import numpy
5 import numpy
6 import h5py
6 import h5py
7
7
8 import schainpy.admin
8 import schainpy.admin
9 from schainpy.model.data.jrodata import *
9 from schainpy.model.data.jrodata import *
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 from schainpy.utils import log
12 from schainpy.utils import log
13
13
14
14
15 class HDFReader(Reader, ProcessingUnit):
15 class HDFReader(Reader, ProcessingUnit):
16 """Processing unit to read HDF5 format files
16 """Processing unit to read HDF5 format files
17
17
18 This unit reads HDF5 files created with `HDFWriter` operation contains
18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 attributes.
20 attributes.
21 It is possible to read any HDF5 file by given the structure in the `description`
21 It is possible to read any HDF5 file by given the structure in the `description`
22 parameter, also you can add extra values to metadata with the parameter `extras`.
22 parameter, also you can add extra values to metadata with the parameter `extras`.
23
23
24 Parameters:
24 Parameters:
25 -----------
25 -----------
26 path : str
26 path : str
27 Path where files are located.
27 Path where files are located.
28 startDate : date
28 startDate : date
29 Start date of the files
29 Start date of the files
30 endDate : list
30 endDate : list
31 End date of the files
31 End date of the files
32 startTime : time
32 startTime : time
33 Start time of the files
33 Start time of the files
34 endTime : time
34 endTime : time
35 End time of the files
35 End time of the files
36 description : dict, optional
36 description : dict, optional
37 Dictionary with the description of the HDF5 file
37 Dictionary with the description of the HDF5 file
38 extras : dict, optional
38 extras : dict, optional
39 Dictionary with extra metadata to be be added to `dataOut`
39 Dictionary with extra metadata to be be added to `dataOut`
40
40
41 Examples
41 Examples
42 --------
42 --------
43
43
44 desc = {
44 desc = {
45 'Data': {
45 'Data': {
46 'data_output': ['u', 'v', 'w'],
46 'data_output': ['u', 'v', 'w'],
47 'utctime': 'timestamps',
47 'utctime': 'timestamps',
48 } ,
48 } ,
49 'Metadata': {
49 'Metadata': {
50 'heightList': 'heights'
50 'heightList': 'heights'
51 }
51 }
52 }
52 }
53
53
54 desc = {
54 desc = {
55 'Data': {
55 'Data': {
56 'data_output': 'winds',
56 'data_output': 'winds',
57 'utctime': 'timestamps'
57 'utctime': 'timestamps'
58 },
58 },
59 'Metadata': {
59 'Metadata': {
60 'heightList': 'heights'
60 'heightList': 'heights'
61 }
61 }
62 }
62 }
63
63
64 extras = {
64 extras = {
65 'timeZone': 300
65 'timeZone': 300
66 }
66 }
67
67
68 reader = project.addReadUnit(
68 reader = project.addReadUnit(
69 name='HDFReader',
69 name='HDFReader',
70 path='/path/to/files',
70 path='/path/to/files',
71 startDate='2019/01/01',
71 startDate='2019/01/01',
72 endDate='2019/01/31',
72 endDate='2019/01/31',
73 startTime='00:00:00',
73 startTime='00:00:00',
74 endTime='23:59:59',
74 endTime='23:59:59',
75 # description=json.dumps(desc),
75 # description=json.dumps(desc),
76 # extras=json.dumps(extras),
76 # extras=json.dumps(extras),
77 )
77 )
78
78
79 """
79 """
80
80
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82
82
83 def __init__(self):
83 def __init__(self):
84 ProcessingUnit.__init__(self)
84 ProcessingUnit.__init__(self)
85
85
86 self.ext = ".hdf5"
86 self.ext = ".hdf5"
87 self.optchar = "D"
87 self.optchar = "D"
88 self.meta = {}
88 self.meta = {}
89 self.data = {}
89 self.data = {}
90 self.open_file = h5py.File
90 self.open_file = h5py.File
91 self.open_mode = 'r'
91 self.open_mode = 'r'
92 self.description = {}
92 self.description = {}
93 self.extras = {}
93 self.extras = {}
94 self.filefmt = "*%Y%j***"
94 self.filefmt = "*%Y%j***"
95 self.folderfmt = "*%Y%j"
95 self.folderfmt = "*%Y%j"
96 self.utcoffset = 0
96 self.utcoffset = 0
97
97 self.flagUpdateDataOut = False
98 self.dataOut = Parameters()
98 self.dataOut = Parameters()
99 self.dataOut.error=False ## NOTE: Importante definir esto antes inicio
99 self.dataOut.error=False ## NOTE: Importante definir esto antes inicio
100 self.dataOut.flagNoData = True
100 self.dataOut.flagNoData = True
101
101
102 def setup(self, **kwargs):
102 def setup(self, **kwargs):
103
103
104 self.set_kwargs(**kwargs)
104 self.set_kwargs(**kwargs)
105 if not self.ext.startswith('.'):
105 if not self.ext.startswith('.'):
106 self.ext = '.{}'.format(self.ext)
106 self.ext = '.{}'.format(self.ext)
107
107
108 if self.online:
108 if self.online:
109 log.log("Searching files in online mode...", self.name)
109 log.log("Searching files in online mode...", self.name)
110
110
111 for nTries in range(self.nTries):
111 for nTries in range(self.nTries):
112 fullpath = self.searchFilesOnLine(self.path, self.startDate,
112 fullpath = self.searchFilesOnLine(self.path, self.startDate,
113 self.endDate, self.expLabel, self.ext, self.walk,
113 self.endDate, self.expLabel, self.ext, self.walk,
114 self.filefmt, self.folderfmt)
114 self.filefmt, self.folderfmt)
115 pathname, filename = os.path.split(fullpath)
115 pathname, filename = os.path.split(fullpath)
116
116
117 try:
117 try:
118 fullpath = next(fullpath)
118 fullpath = next(fullpath)
119
119
120 except:
120 except:
121 fullpath = None
121 fullpath = None
122
122
123 if fullpath:
123 if fullpath:
124 break
124 break
125
125
126 log.warning(
126 log.warning(
127 'Waiting {} sec for a valid file in {}: try {} ...'.format(
127 'Waiting {} sec for a valid file in {}: try {} ...'.format(
128 self.delay, self.path, nTries + 1),
128 self.delay, self.path, nTries + 1),
129 self.name)
129 self.name)
130 time.sleep(self.delay)
130 time.sleep(self.delay)
131
131
132 if not(fullpath):
132 if not(fullpath):
133 raise schainpy.admin.SchainError(
133 raise schainpy.admin.SchainError(
134 'There isn\'t any valid file in {}'.format(self.path))
134 'There isn\'t any valid file in {}'.format(self.path))
135
135
136 pathname, filename = os.path.split(fullpath)
136 pathname, filename = os.path.split(fullpath)
137 self.year = int(filename[1:5])
137 self.year = int(filename[1:5])
138 self.doy = int(filename[5:8])
138 self.doy = int(filename[5:8])
139 self.set = int(filename[8:11]) - 1
139 self.set = int(filename[8:11]) - 1
140 else:
140 else:
141 log.log("Searching files in {}".format(self.path), self.name)
141 log.log("Searching files in {}".format(self.path), self.name)
142 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
142 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
143 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
143 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
144
144
145 self.setNextFile()
145 self.setNextFile()
146
146
147
147
148
148
149
149
150 # def readFirstHeader(self):
151 # '''Read metadata and data'''
152 #
153 # self.__readMetadata2()
154 # self.__readData()
155 # self.__setBlockList()
156 #
157 # for attr in self.meta:
158 # setattr(self.dataOut, attr, self.meta[attr])
159 # self.blockIndex = 0
160 #
161 # return
162
150 def readFirstHeader(self):
163 def readFirstHeader(self):
151 '''Read metadata and data'''
164 '''Read metadata and data'''
152
165
153 self.__readMetadata()
166 self.__readMetadata2()
154 self.__readData()
167 self.__readData()
155 self.__setBlockList()
168 self.__setBlockList()
156
157 for attr in self.meta:
169 for attr in self.meta:
158 setattr(self.dataOut, attr, self.meta[attr])
170 if "processingHeaderObj" in attr:
171 self.flagUpdateDataOut=True
172 at = attr.split('.')
173 if len(at) > 1:
174 setattr(eval("self.dataOut."+at[0]),at[1], self.meta[attr])
175 else:
176 setattr(self.dataOut, attr, self.meta[attr])
159 self.blockIndex = 0
177 self.blockIndex = 0
160
178
179 if self.flagUpdateDataOut:
180 self.updateDataOut()
181
161 return
182 return
162
183
184 def updateDataOut(self):
185 self.dataOut.azimuthList = self.dataOut.processingHeaderObj.azimuthList
186 self.dataOut.elevationList = self.dataOut.processingHeaderObj.elevationList
187 self.dataOut.heightList = self.dataOut.processingHeaderObj.heightList
188 self.dataOut.ippSeconds = self.dataOut.processingHeaderObj.ipp
189 self.dataOut.elevationList = self.dataOut.processingHeaderObj.elevationList
190 self.dataOut.channelList = self.dataOut.processingHeaderObj.channelList
191 self.dataOut.nCohInt = self.dataOut.processingHeaderObj.nCohInt
192 self.dataOut.nFFTPoints = self.dataOut.processingHeaderObj.nFFTPoints
193 self.flagUpdateDataOut = False
194 self.dataOut.frequency = self.dataOut.radarControllerHeaderObj.frequency
195 #self.dataOut.heightList = self.dataOut.processingHeaderObj.heightList
196
197
198 return
199
200
163 def __setBlockList(self):
201 def __setBlockList(self):
164 '''
202 '''
165 Selects the data within the times defined
203 Selects the data within the times defined
166
204
167 self.fp
205 self.fp
168 self.startTime
206 self.startTime
169 self.endTime
207 self.endTime
170 self.blockList
208 self.blockList
171 self.blocksPerFile
209 self.blocksPerFile
172
210
173 '''
211 '''
174
212
175 startTime = self.startTime
213 startTime = self.startTime
176 endTime = self.endTime
214 endTime = self.endTime
177 thisUtcTime = self.data['utctime'] + self.utcoffset
215 thisUtcTime = self.data['utctime'] + self.utcoffset
178 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
216 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
179 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
217 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
180 self.startFileDatetime = thisDatetime
218 self.startFileDatetime = thisDatetime
181 thisDate = thisDatetime.date()
219 thisDate = thisDatetime.date()
182 thisTime = thisDatetime.time()
220 thisTime = thisDatetime.time()
183
221
184 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
222 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
185 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
223 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
186
224
187 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
225 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
188
226
189 self.blockList = ind
227 self.blockList = ind
190 self.blocksPerFile = len(ind)
228 self.blocksPerFile = len(ind)
191 self.blocksPerFile = len(thisUtcTime)
229 self.blocksPerFile = len(thisUtcTime)
192 return
230 return
193
231
194 def __readMetadata(self):
232 def __readMetadata(self):
195 '''
233 '''
196 Reads Metadata
234 Reads Metadata
197 '''
235 '''
198
236
199 meta = {}
237 meta = {}
200
238
201 if self.description:
239 if self.description:
202 for key, value in self.description['Metadata'].items():
240 for key, value in self.description['Metadata'].items():
203 meta[key] = self.fp[value][()]
241 meta[key] = self.fp[value][()]
204 else:
242 else:
205 grp = self.fp['Metadata']
243 grp = self.fp['Metadata']
206 for name in grp:
244 for name in grp:
207 meta[name] = grp[name][()]
245 meta[name] = grp[name][()]
208
246
209 if self.extras:
247 if self.extras:
210 for key, value in self.extras.items():
248 for key, value in self.extras.items():
211 meta[key] = value
249 meta[key] = value
212 self.meta = meta
250 self.meta = meta
213
251
214 return
252 return
215
253
216
254
255 def __readMetadata2(self):
256 '''
257 Reads Metadata
258 '''
259
260 meta = {}
261
262 if self.description:
263 for key, value in self.description['Metadata'].items():
264 meta[key] = self.fp[value][()]
265 else:
266 grp = self.fp['Metadata']
267 for item in grp.values():
268 name = item.name
269 if isinstance(item, h5py.Dataset):
270 name = name.split("/")[-1]
271 meta[name] = item[()]
272 else:
273 grp2 = self.fp[name]
274 Obj = name.split("/")[-1]
275
276 for item2 in grp2.values():
277 name2 = Obj+"."+item2.name.split("/")[-1]
278 meta[name2] = item2[()]
279
280 if self.extras:
281 for key, value in self.extras.items():
282 meta[key] = value
283 self.meta = meta
284 #print(meta)
285 return
286
217
287
218 def checkForRealPath(self, nextFile, nextDay):
288 def checkForRealPath(self, nextFile, nextDay):
219
289
220 # print("check FRP")
290 # print("check FRP")
221 # dt = self.startFileDatetime + datetime.timedelta(1)
291 # dt = self.startFileDatetime + datetime.timedelta(1)
222 # filename = '{}.{}{}'.format(self.path, dt.strftime('%Y%m%d'), self.ext)
292 # filename = '{}.{}{}'.format(self.path, dt.strftime('%Y%m%d'), self.ext)
223 # fullfilename = os.path.join(self.path, filename)
293 # fullfilename = os.path.join(self.path, filename)
224 # print("check Path ",fullfilename,filename)
294 # print("check Path ",fullfilename,filename)
225 # if os.path.exists(fullfilename):
295 # if os.path.exists(fullfilename):
226 # return fullfilename, filename
296 # return fullfilename, filename
227 # return None, filename
297 # return None, filename
228 return None,None
298 return None,None
229
299
230 def __readData(self):
300 def __readData(self):
231
301
232 data = {}
302 data = {}
233
303
234 if self.description:
304 if self.description:
235 for key, value in self.description['Data'].items():
305 for key, value in self.description['Data'].items():
236 if isinstance(value, str):
306 if isinstance(value, str):
237 if isinstance(self.fp[value], h5py.Dataset):
307 if isinstance(self.fp[value], h5py.Dataset):
238 data[key] = self.fp[value][()]
308 data[key] = self.fp[value][()]
239 elif isinstance(self.fp[value], h5py.Group):
309 elif isinstance(self.fp[value], h5py.Group):
240 array = []
310 array = []
241 for ch in self.fp[value]:
311 for ch in self.fp[value]:
242 array.append(self.fp[value][ch][()])
312 array.append(self.fp[value][ch][()])
243 data[key] = numpy.array(array)
313 data[key] = numpy.array(array)
244 elif isinstance(value, list):
314 elif isinstance(value, list):
245 array = []
315 array = []
246 for ch in value:
316 for ch in value:
247 array.append(self.fp[ch][()])
317 array.append(self.fp[ch][()])
248 data[key] = numpy.array(array)
318 data[key] = numpy.array(array)
249 else:
319 else:
250 grp = self.fp['Data']
320 grp = self.fp['Data']
251 for name in grp:
321 for name in grp:
252 if isinstance(grp[name], h5py.Dataset):
322 if isinstance(grp[name], h5py.Dataset):
253 array = grp[name][()]
323 array = grp[name][()]
254 elif isinstance(grp[name], h5py.Group):
324 elif isinstance(grp[name], h5py.Group):
255 array = []
325 array = []
256 for ch in grp[name]:
326 for ch in grp[name]:
257 array.append(grp[name][ch][()])
327 array.append(grp[name][ch][()])
258 array = numpy.array(array)
328 array = numpy.array(array)
259 else:
329 else:
260 log.warning('Unknown type: {}'.format(name))
330 log.warning('Unknown type: {}'.format(name))
261
331
262 if name in self.description:
332 if name in self.description:
263 key = self.description[name]
333 key = self.description[name]
264 else:
334 else:
265 key = name
335 key = name
266 data[key] = array
336 data[key] = array
267
337
268 self.data = data
338 self.data = data
269 return
339 return
270
340
271 def getData(self):
341 def getData(self):
272 if not self.isDateTimeInRange(self.startFileDatetime, self.startDate, self.endDate, self.startTime, self.endTime):
342 if not self.isDateTimeInRange(self.startFileDatetime, self.startDate, self.endDate, self.startTime, self.endTime):
273 self.dataOut.flagNoData = True
343 self.dataOut.flagNoData = True
274 self.blockIndex = self.blocksPerFile
344 self.blockIndex = self.blocksPerFile
275 self.dataOut.error = True # TERMINA EL PROGRAMA
345 self.dataOut.error = True # TERMINA EL PROGRAMA
276 return
346 return
277 for attr in self.data:
347 for attr in self.data:
278
348
279 if self.data[attr].ndim == 1:
349 if self.data[attr].ndim == 1:
280 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
350 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
281 else:
351 else:
282 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
352 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
283
353
284
354
285 self.blockIndex += 1
355 self.blockIndex += 1
286
356
287 if self.blockIndex == 1:
357 if self.blockIndex == 1:
288 log.log("Block No. {}/{} -> {}".format(
358 log.log("Block No. {}/{} -> {}".format(
289 self.blockIndex,
359 self.blockIndex,
290 self.blocksPerFile,
360 self.blocksPerFile,
291 self.dataOut.datatime.ctime()), self.name)
361 self.dataOut.datatime.ctime()), self.name)
292 else:
362 else:
293 log.log("Block No. {}/{} ".format(
363 log.log("Block No. {}/{} ".format(
294 self.blockIndex,
364 self.blockIndex,
295 self.blocksPerFile),self.name)
365 self.blocksPerFile),self.name)
296
366
297 if self.blockIndex == self.blocksPerFile:
367 if self.blockIndex == self.blocksPerFile:
298 self.setNextFile()
368 self.setNextFile()
299
369
300 self.dataOut.flagNoData = False
370 self.dataOut.flagNoData = False
301
371
302
372
303 def run(self, **kwargs):
373 def run(self, **kwargs):
304
374
305 if not(self.isConfig):
375 if not(self.isConfig):
306 self.setup(**kwargs)
376 self.setup(**kwargs)
307 self.isConfig = True
377 self.isConfig = True
308
378
309 self.getData()
379 self.getData()
310
380
311 @MPDecorator
381 @MPDecorator
312 class HDFWriter(Operation):
382 class HDFWriter(Operation):
313 """Operation to write HDF5 files.
383 """Operation to write HDF5 files.
314
384
315 The HDF5 file contains by default two groups Data and Metadata where
385 The HDF5 file contains by default two groups Data and Metadata where
316 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
386 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
317 parameters, data attributes are normaly time dependent where the metadata
387 parameters, data attributes are normaly time dependent where the metadata
318 are not.
388 are not.
319 It is possible to customize the structure of the HDF5 file with the
389 It is possible to customize the structure of the HDF5 file with the
320 optional description parameter see the examples.
390 optional description parameter see the examples.
321
391
322 Parameters:
392 Parameters:
323 -----------
393 -----------
324 path : str
394 path : str
325 Path where files will be saved.
395 Path where files will be saved.
326 blocksPerFile : int
396 blocksPerFile : int
327 Number of blocks per file
397 Number of blocks per file
328 metadataList : list
398 metadataList : list
329 List of the dataOut attributes that will be saved as metadata
399 List of the dataOut attributes that will be saved as metadata
330 dataList : int
400 dataList : int
331 List of the dataOut attributes that will be saved as data
401 List of the dataOut attributes that will be saved as data
332 setType : bool
402 setType : bool
333 If True the name of the files corresponds to the timestamp of the data
403 If True the name of the files corresponds to the timestamp of the data
334 description : dict, optional
404 description : dict, optional
335 Dictionary with the desired description of the HDF5 file
405 Dictionary with the desired description of the HDF5 file
336
406
337 Examples
407 Examples
338 --------
408 --------
339
409
340 desc = {
410 desc = {
341 'data_output': {'winds': ['z', 'w', 'v']},
411 'data_output': {'winds': ['z', 'w', 'v']},
342 'utctime': 'timestamps',
412 'utctime': 'timestamps',
343 'heightList': 'heights'
413 'heightList': 'heights'
344 }
414 }
345 desc = {
415 desc = {
346 'data_output': ['z', 'w', 'v'],
416 'data_output': ['z', 'w', 'v'],
347 'utctime': 'timestamps',
417 'utctime': 'timestamps',
348 'heightList': 'heights'
418 'heightList': 'heights'
349 }
419 }
350 desc = {
420 desc = {
351 'Data': {
421 'Data': {
352 'data_output': 'winds',
422 'data_output': 'winds',
353 'utctime': 'timestamps'
423 'utctime': 'timestamps'
354 },
424 },
355 'Metadata': {
425 'Metadata': {
356 'heightList': 'heights'
426 'heightList': 'heights'
357 }
427 }
358 }
428 }
359
429
360 writer = proc_unit.addOperation(name='HDFWriter')
430 writer = proc_unit.addOperation(name='HDFWriter')
361 writer.addParameter(name='path', value='/path/to/file')
431 writer.addParameter(name='path', value='/path/to/file')
362 writer.addParameter(name='blocksPerFile', value='32')
432 writer.addParameter(name='blocksPerFile', value='32')
363 writer.addParameter(name='metadataList', value='heightList,timeZone')
433 writer.addParameter(name='metadataList', value='heightList,timeZone')
364 writer.addParameter(name='dataList',value='data_output,utctime')
434 writer.addParameter(name='dataList',value='data_output,utctime')
365 # writer.addParameter(name='description',value=json.dumps(desc))
435 # writer.addParameter(name='description',value=json.dumps(desc))
366
436
367 """
437 """
368
438
369 ext = ".hdf5"
439 ext = ".hdf5"
370 optchar = "D"
440 optchar = "D"
371 filename = None
441 filename = None
372 path = None
442 path = None
373 setFile = None
443 setFile = None
374 fp = None
444 fp = None
375 firsttime = True
445 firsttime = True
376 #Configurations
446 #Configurations
377 blocksPerFile = None
447 blocksPerFile = None
378 blockIndex = None
448 blockIndex = None
379 dataOut = None #eval ??????
449 dataOut = None #eval ??????
380 #Data Arrays
450 #Data Arrays
381 dataList = None
451 dataList = None
382 metadataList = None
452 metadataList = None
383 currentDay = None
453 currentDay = None
384 lastTime = None
454 lastTime = None
385 timeZone = "ut"
455 timeZone = "ut"
386 hourLimit = 3
456 hourLimit = 3
387 breakDays = True
457 breakDays = True
388
458
389 def __init__(self):
459 def __init__(self):
390
460
391 Operation.__init__(self)
461 Operation.__init__(self)
392
462
393
463
394 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None,
464 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None,
395 description={},timeZone = "ut",hourLimit = 3, breakDays=True):
465 description={},timeZone = "ut",hourLimit = 3, breakDays=True):
396 self.path = path
466 self.path = path
397 self.blocksPerFile = blocksPerFile
467 self.blocksPerFile = blocksPerFile
398 self.metadataList = metadataList
468 self.metadataList = metadataList
399 self.dataList = [s.strip() for s in dataList]
469 self.dataList = [s.strip() for s in dataList]
400 self.setType = setType
470 self.setType = setType
401 self.description = description
471 self.description = description
402 self.timeZone = timeZone
472 self.timeZone = timeZone
403 self.hourLimit = hourLimit
473 self.hourLimit = hourLimit
404 self.breakDays = breakDays
474 self.breakDays = breakDays
405
475
406 if self.metadataList is None:
476 if self.metadataList is None:
407 self.metadataList = self.dataOut.metadata_list
477 self.metadataList = self.dataOut.metadata_list
408
478
479 self.metadataList = list(set(self.metadataList))
480
409 tableList = []
481 tableList = []
410 dsList = []
482 dsList = []
411
483
412 for i in range(len(self.dataList)):
484 for i in range(len(self.dataList)):
413 dsDict = {}
485 dsDict = {}
414 if hasattr(self.dataOut, self.dataList[i]):
486 if hasattr(self.dataOut, self.dataList[i]):
415 dataAux = getattr(self.dataOut, self.dataList[i])
487 dataAux = getattr(self.dataOut, self.dataList[i])
416 dsDict['variable'] = self.dataList[i]
488 dsDict['variable'] = self.dataList[i]
417 else:
489 else:
418 log.warning('Attribute {} not found in dataOut'.format(self.dataList[i]),self.name)
490 log.warning('Attribute {} not found in dataOut'.format(self.dataList[i]),self.name)
419 continue
491 continue
420
492
421 if dataAux is None:
493 if dataAux is None:
422 continue
494 continue
423 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
495 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
424 dsDict['nDim'] = 0
496 dsDict['nDim'] = 0
425 else:
497 else:
426 dsDict['nDim'] = len(dataAux.shape)
498 dsDict['nDim'] = len(dataAux.shape)
427 dsDict['shape'] = dataAux.shape
499 dsDict['shape'] = dataAux.shape
428 dsDict['dsNumber'] = dataAux.shape[0]
500 dsDict['dsNumber'] = dataAux.shape[0]
429 dsDict['dtype'] = dataAux.dtype
501 dsDict['dtype'] = dataAux.dtype
430
502
431 dsList.append(dsDict)
503 dsList.append(dsDict)
432
504
433 self.blockIndex = 0
505 self.blockIndex = 0
434 self.dsList = dsList
506 self.dsList = dsList
435 self.currentDay = self.dataOut.datatime.date()
507 self.currentDay = self.dataOut.datatime.date()
436
508
437
509
438 def timeFlag(self):
510 def timeFlag(self):
439 currentTime = self.dataOut.utctime
511 currentTime = self.dataOut.utctime
440 timeTuple = None
512 timeTuple = None
441 if self.timeZone == "lt":
513 if self.timeZone == "lt":
442 timeTuple = time.localtime(currentTime)
514 timeTuple = time.localtime(currentTime)
443 else :
515 else :
444 timeTuple = time.gmtime(currentTime)
516 timeTuple = time.gmtime(currentTime)
445
517
446 dataDay = timeTuple.tm_yday
518 dataDay = timeTuple.tm_yday
447
519
448 if self.lastTime is None:
520 if self.lastTime is None:
449 self.lastTime = currentTime
521 self.lastTime = currentTime
450 self.currentDay = dataDay
522 self.currentDay = dataDay
451 return False
523 return False
452
524
453 timeDiff = currentTime - self.lastTime
525 timeDiff = currentTime - self.lastTime
454
526
455 #Si el dia es diferente o si la diferencia entre un dato y otro supera self.hourLimit
527 #Si el dia es diferente o si la diferencia entre un dato y otro supera self.hourLimit
456 if (dataDay != self.currentDay) and self.breakDays:
528 if (dataDay != self.currentDay) and self.breakDays:
457 self.currentDay = dataDay
529 self.currentDay = dataDay
458 return True
530 return True
459 elif timeDiff > self.hourLimit*60*60:
531 elif timeDiff > self.hourLimit*60*60:
460 self.lastTime = currentTime
532 self.lastTime = currentTime
461 return True
533 return True
462 else:
534 else:
463 self.lastTime = currentTime
535 self.lastTime = currentTime
464 return False
536 return False
465
537
466 def run(self, dataOut,**kwargs):
538 def run(self, dataOut,**kwargs):
467
539
468 self.dataOut = dataOut
540 self.dataOut = dataOut
469 #print(self.dataOut.radarControllerHeaderObj.toString())
541 #print(self.dataOut.radarControllerHeaderObj.toString())
470 if not(self.isConfig):
542 if not(self.isConfig):
471 self.setup(**kwargs)
543 self.setup(**kwargs)
472
544
473 self.isConfig = True
545 self.isConfig = True
474 self.setNextFile()
546 self.setNextFile()
475
547
476 self.putData()
548 self.putData()
477
549
478 #return self.dataOut
550 #return self.dataOut
479
551
480 def setNextFile(self):
552 def setNextFile(self):
481
553
482 ext = self.ext
554 ext = self.ext
483 path = self.path
555 path = self.path
484 setFile = self.setFile
556 setFile = self.setFile
485 timeTuple = None
557 timeTuple = None
486 if self.timeZone == "lt":
558 if self.timeZone == "lt":
487 timeTuple = time.localtime(self.dataOut.utctime)
559 timeTuple = time.localtime(self.dataOut.utctime)
488 elif self.timeZone == "ut":
560 elif self.timeZone == "ut":
489 timeTuple = time.gmtime(self.dataOut.utctime)
561 timeTuple = time.gmtime(self.dataOut.utctime)
490 #print("path: ",timeTuple)
562 #print("path: ",timeTuple)
491 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
563 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
492 fullpath = os.path.join(path, subfolder)
564 fullpath = os.path.join(path, subfolder)
493
565
494 if os.path.exists(fullpath):
566 if os.path.exists(fullpath):
495 filesList = os.listdir(fullpath)
567 filesList = os.listdir(fullpath)
496 filesList = [k for k in filesList if k.startswith(self.optchar)]
568 filesList = [k for k in filesList if k.startswith(self.optchar)]
497 if len( filesList ) > 0:
569 if len( filesList ) > 0:
498 filesList = sorted(filesList, key=str.lower)
570 filesList = sorted(filesList, key=str.lower)
499 filen = filesList[-1]
571 filen = filesList[-1]
500 # el filename debera tener el siguiente formato
572 # el filename debera tener el siguiente formato
501 # 0 1234 567 89A BCDE (hex)
573 # 0 1234 567 89A BCDE (hex)
502 # x YYYY DDD SSS .ext
574 # x YYYY DDD SSS .ext
503 if isNumber(filen[8:11]):
575 if isNumber(filen[8:11]):
504 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
576 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
505 else:
577 else:
506 setFile = -1
578 setFile = -1
507 else:
579 else:
508 setFile = -1 #inicializo mi contador de seteo
580 setFile = -1 #inicializo mi contador de seteo
509 else:
581 else:
510 os.makedirs(fullpath)
582 os.makedirs(fullpath)
511 setFile = -1 #inicializo mi contador de seteo
583 setFile = -1 #inicializo mi contador de seteo
512
584
513 if self.setType is None:
585 if self.setType is None:
514 setFile += 1
586 setFile += 1
515 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
587 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
516 timeTuple.tm_year,
588 timeTuple.tm_year,
517 timeTuple.tm_yday,
589 timeTuple.tm_yday,
518 setFile,
590 setFile,
519 ext )
591 ext )
520 else:
592 else:
521 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
593 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
522 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
594 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
523 timeTuple.tm_year,
595 timeTuple.tm_year,
524 timeTuple.tm_yday,
596 timeTuple.tm_yday,
525 setFile,
597 setFile,
526 ext )
598 ext )
527
599
528 self.filename = os.path.join( path, subfolder, file )
600 self.filename = os.path.join( path, subfolder, file )
529
601
530
602
531
603
532 def getLabel(self, name, x=None):
604 def getLabel(self, name, x=None):
533
605
534 if x is None:
606 if x is None:
535 if 'Data' in self.description:
607 if 'Data' in self.description:
536 data = self.description['Data']
608 data = self.description['Data']
537 if 'Metadata' in self.description:
609 if 'Metadata' in self.description:
538 data.update(self.description['Metadata'])
610 data.update(self.description['Metadata'])
539 else:
611 else:
540 data = self.description
612 data = self.description
541 if name in data:
613 if name in data:
542 if isinstance(data[name], str):
614 if isinstance(data[name], str):
543 return data[name]
615 return data[name]
544 elif isinstance(data[name], list):
616 elif isinstance(data[name], list):
545 return None
617 return None
546 elif isinstance(data[name], dict):
618 elif isinstance(data[name], dict):
547 for key, value in data[name].items():
619 for key, value in data[name].items():
548 return key
620 return key
549 return name
621 return name
550 else:
622 else:
551 if 'Metadata' in self.description:
623 if 'Metadata' in self.description:
552 meta = self.description['Metadata']
624 meta = self.description['Metadata']
553 else:
625 else:
554 meta = self.description
626 meta = self.description
555 if name in meta:
627 if name in meta:
556 if isinstance(meta[name], list):
628 if isinstance(meta[name], list):
557 return meta[name][x]
629 return meta[name][x]
558 elif isinstance(meta[name], dict):
630 elif isinstance(meta[name], dict):
559 for key, value in meta[name].items():
631 for key, value in meta[name].items():
560 return value[x]
632 return value[x]
561 if 'cspc' in name:
633 if 'cspc' in name:
562 return 'pair{:02d}'.format(x)
634 return 'pair{:02d}'.format(x)
563 else:
635 else:
564 return 'channel{:02d}'.format(x)
636 return 'channel{:02d}'.format(x)
565
637
566 def writeMetadata(self, fp):
638 def writeMetadata(self, fp):
567
639
568 if self.description:
640 if self.description:
569 if 'Metadata' in self.description:
641 if 'Metadata' in self.description:
570 grp = fp.create_group('Metadata')
642 grp = fp.create_group('Metadata')
571 else:
643 else:
572 grp = fp
644 grp = fp
573 else:
645 else:
574 grp = fp.create_group('Metadata')
646 grp = fp.create_group('Metadata')
575
647
576 for i in range(len(self.metadataList)):
648 for i in range(len(self.metadataList)):
577 attribute = self.metadataList[i]
649 attribute = self.metadataList[i]
578 if not hasattr(self.dataOut,attribute ):
650 if not hasattr(self.dataOut,attribute ):
579 log.warning('Metadata: `{}` not found'.format(attribute), self.name)
651 log.warning('Metadata: `{}` not found'.format(attribute), self.name)
580 continue
652 continue
581 value = getattr(self.dataOut, attribute)
653 value = getattr(self.dataOut, attribute)
582 if isinstance(value, bool):
654 if isinstance(value, bool):
583 if value is True:
655 if value is True:
584 value = 1
656 value = 1
585 else:
657 else:
586 value = 0
658 value = 0
587 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
659 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
588 return
660 return
589
661
662 def writeMetadata2(self, fp):
663
664 if self.description:
665 if 'Metadata' in self.description:
666 grp = fp.create_group('Metadata')
667 else:
668 grp = fp
669 else:
670 grp = fp.create_group('Metadata')
671
672
673 for i in range(len(self.metadataList)):
674
675 attribute = self.metadataList[i]
676 attr = attribute.split('.')
677 if len(attr) > 1:
678 if not hasattr(eval("self.dataOut."+attr[0]),attr[1]):
679 log.warning('Metadata: {}.{} not found'.format(attr[0],attr[1]), self.name)
680 continue
681 value = getattr(eval("self.dataOut."+attr[0]),attr[1])
682 if isinstance(value, bool):
683 if value is True:
684 value = 1
685 else:
686 value = 0
687 if isinstance(value,type(None)):
688 log.warning("Invalid value detected, {} is None".format(attribute), self.name)
689 value = 0
690 grp2 = None
691 if not 'Metadata/'+attr[0] in fp:
692 grp2 = fp.create_group('Metadata/'+attr[0])
693 else:
694 grp2 = fp['Metadata/'+attr[0]]
695 #print("attribute: ", attribute, value)
696 grp2.create_dataset(attr[1], data=value)
697
698 else:
699 if not hasattr(self.dataOut, attr[0] ):
700 log.warning('Metadata: `{}` not found'.format(attribute), self.name)
701 continue
702 value = getattr(self.dataOut, attr[0])
703 if isinstance(value, bool):
704 if value is True:
705 value = 1
706 else:
707 value = 0
708 grp.create_dataset(self.getLabel(attribute), data=value)
709
710 return
711
590 def writeData(self, fp):
712 def writeData(self, fp):
591
713
592 if self.description:
714 if self.description:
593 if 'Data' in self.description:
715 if 'Data' in self.description:
594 grp = fp.create_group('Data')
716 grp = fp.create_group('Data')
595 else:
717 else:
596 grp = fp
718 grp = fp
597 else:
719 else:
598 grp = fp.create_group('Data')
720 grp = fp.create_group('Data')
599
721
600 dtsets = []
722 dtsets = []
601 data = []
723 data = []
602
724
603 for dsInfo in self.dsList:
725 for dsInfo in self.dsList:
604 if dsInfo['nDim'] == 0:
726 if dsInfo['nDim'] == 0:
605 ds = grp.create_dataset(
727 ds = grp.create_dataset(
606 self.getLabel(dsInfo['variable']),
728 self.getLabel(dsInfo['variable']),
607 (self.blocksPerFile, ),
729 (self.blocksPerFile, ),
608 chunks=True,
730 chunks=True,
609 dtype=numpy.float64)
731 dtype=numpy.float64)
610 dtsets.append(ds)
732 dtsets.append(ds)
611 data.append((dsInfo['variable'], -1))
733 data.append((dsInfo['variable'], -1))
612 else:
734 else:
613 label = self.getLabel(dsInfo['variable'])
735 label = self.getLabel(dsInfo['variable'])
614 if label is not None:
736 if label is not None:
615 sgrp = grp.create_group(label)
737 sgrp = grp.create_group(label)
616 else:
738 else:
617 sgrp = grp
739 sgrp = grp
618 for i in range(dsInfo['dsNumber']):
740 for i in range(dsInfo['dsNumber']):
619 ds = sgrp.create_dataset(
741 ds = sgrp.create_dataset(
620 self.getLabel(dsInfo['variable'], i),
742 self.getLabel(dsInfo['variable'], i),
621 (self.blocksPerFile, ) + dsInfo['shape'][1:],
743 (self.blocksPerFile, ) + dsInfo['shape'][1:],
622 chunks=True,
744 chunks=True,
623 dtype=dsInfo['dtype'])
745 dtype=dsInfo['dtype'])
624 dtsets.append(ds)
746 dtsets.append(ds)
625 data.append((dsInfo['variable'], i))
747 data.append((dsInfo['variable'], i))
626 fp.flush()
748 fp.flush()
627
749
628 log.log('Creating file: {}'.format(fp.filename), self.name)
750 log.log('Creating file: {}'.format(fp.filename), self.name)
629
751
630 self.ds = dtsets
752 self.ds = dtsets
631 self.data = data
753 self.data = data
632 self.firsttime = True
754 self.firsttime = True
633
755
634 return
756 return
635
757
636 def putData(self):
758 def putData(self):
637
759
638 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
760 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
639 self.closeFile()
761 self.closeFile()
640 self.setNextFile()
762 self.setNextFile()
641 self.dataOut.flagNoData = False
763 self.dataOut.flagNoData = False
642 self.blockIndex = 0
764 self.blockIndex = 0
643 return
765 return
644
766
645
767
646
768
647 if self.blockIndex == 0:
769 if self.blockIndex == 0:
648 #Escribir metadata Aqui???
770 #Escribir metadata Aqui???
649 #Setting HDF5 File
771 #Setting HDF5 File
650 self.fp = h5py.File(self.filename, 'w')
772 self.fp = h5py.File(self.filename, 'w')
651 #write metadata
773 #write metadata
652 self.writeMetadata(self.fp)
774 self.writeMetadata2(self.fp)
653 #Write data
775 #Write data
654 self.writeData(self.fp)
776 self.writeData(self.fp)
655 log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
777 log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
656 elif (self.blockIndex % 10 ==0):
778 elif (self.blockIndex % 10 ==0):
657 log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
779 log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
658 else:
780 else:
659
781
660 log.log('Block No. {}/{}'.format(self.blockIndex+1, self.blocksPerFile), self.name)
782 log.log('Block No. {}/{}'.format(self.blockIndex+1, self.blocksPerFile), self.name)
661
783
662 for i, ds in enumerate(self.ds):
784 for i, ds in enumerate(self.ds):
663 attr, ch = self.data[i]
785 attr, ch = self.data[i]
664 if ch == -1:
786 if ch == -1:
665 ds[self.blockIndex] = getattr(self.dataOut, attr)
787 ds[self.blockIndex] = getattr(self.dataOut, attr)
666 else:
788 else:
667 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
789 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
668
790
669 self.blockIndex += 1
791 self.blockIndex += 1
670
792
671 self.fp.flush()
793 self.fp.flush()
672 self.dataOut.flagNoData = True
794 self.dataOut.flagNoData = True
673
795
674
796
675 def closeFile(self):
797 def closeFile(self):
676
798
677 if self.blockIndex != self.blocksPerFile:
799 if self.blockIndex != self.blocksPerFile:
678 for ds in self.ds:
800 for ds in self.ds:
679 ds.resize(self.blockIndex, axis=0)
801 ds.resize(self.blockIndex, axis=0)
680
802
681 if self.fp:
803 if self.fp:
682 self.fp.flush()
804 self.fp.flush()
683 self.fp.close()
805 self.fp.close()
684
806
685 def close(self):
807 def close(self):
686
808
687 self.closeFile()
809 self.closeFile()
@@ -1,523 +1,661
1 """
1 """
2 Utilities for IO modules
2 Utilities for IO modules
3 @modified: Joab Apaza
3 @modified: Joab Apaza
4 @email: roj-op01@igp.gob.pe, joab.apaza32@gmail.com
4 @email: roj-op01@igp.gob.pe, joab.apaza32@gmail.com
5
5
6 """
6 """
7 ################################################################################
7 ################################################################################
8 ################################################################################
8 ################################################################################
9 import os
9 import os
10 from datetime import datetime
10 from datetime import datetime
11 import numpy
11 import numpy
12
12
13 from schainpy.model.data.jrodata import Parameters
13 from schainpy.model.data.jrodata import Parameters
14 import itertools
14 import itertools
15 import numpy
15 import numpy
16 import h5py
16 import h5py
17 import re
17 import re
18 import time
18 import time
19 ################################################################################
19 ################################################################################
20 ################################################################################
20 ################################################################################
21 ################################################################################
21 ################################################################################
22 def folder_in_range(folder, start_date, end_date, pattern):
22 def folder_in_range(folder, start_date, end_date, pattern):
23 """
23 """
24 Check whether folder is bettwen start_date and end_date
24 Check whether folder is bettwen start_date and end_date
25
25
26 Args:
26 Args:
27 folder (str): Folder to check
27 folder (str): Folder to check
28 start_date (date): Initial date
28 start_date (date): Initial date
29 end_date (date): Final date
29 end_date (date): Final date
30 pattern (str): Datetime format of the folder
30 pattern (str): Datetime format of the folder
31 Returns:
31 Returns:
32 bool: True for success, False otherwise
32 bool: True for success, False otherwise
33 """
33 """
34 try:
34 try:
35 dt = datetime.strptime(folder, pattern)
35 dt = datetime.strptime(folder, pattern)
36 except:
36 except:
37 raise ValueError('Folder {} does not match {} format'.format(folder, pattern))
37 raise ValueError('Folder {} does not match {} format'.format(folder, pattern))
38 return start_date <= dt.date() <= end_date
38 return start_date <= dt.date() <= end_date
39
39
40 ################################################################################
40 ################################################################################
41 ################################################################################
41 ################################################################################
42 ################################################################################
42 ################################################################################
43 def getHei_index( minHei, maxHei, heightList):
43 def getHei_index( minHei, maxHei, heightList):
44 if (minHei < heightList[0]):
44 if (minHei < heightList[0]):
45 minHei = heightList[0]
45 minHei = heightList[0]
46
46
47 if (maxHei > heightList[-1]):
47 if (maxHei > heightList[-1]):
48 maxHei = heightList[-1]
48 maxHei = heightList[-1]
49
49
50 minIndex = 0
50 minIndex = 0
51 maxIndex = 0
51 maxIndex = 0
52 heights = numpy.asarray(heightList)
52 heights = numpy.asarray(heightList)
53
53
54 inda = numpy.where(heights >= minHei)
54 inda = numpy.where(heights >= minHei)
55 indb = numpy.where(heights <= maxHei)
55 indb = numpy.where(heights <= maxHei)
56
56
57 try:
57 try:
58 minIndex = inda[0][0]
58 minIndex = inda[0][0]
59 except:
59 except:
60 minIndex = 0
60 minIndex = 0
61
61
62 try:
62 try:
63 maxIndex = indb[0][-1]
63 maxIndex = indb[0][-1]
64 except:
64 except:
65 maxIndex = len(heightList)
65 maxIndex = len(heightList)
66 return minIndex,maxIndex
66 return minIndex,maxIndex
67
67
68 ################################################################################
68 ################################################################################
69 ################################################################################
69 ################################################################################
70 ################################################################################
70 ################################################################################
71 class MergeH5(object):
71 class MergeH5(object):
72 """Processing unit to read HDF5 format files
72 """Processing unit to read HDF5 format files
73
73
74 This unit reads HDF5 files created with `HDFWriter` operation when channels area
74 This unit reads HDF5 files created with `HDFWriter` operation when channels area
75 processed by separated. Then merge all channels in a single files.
75 processed by separated. Then merge all channels in a single files.
76
76
77 "example"
77 "example"
78 nChannels = 4
78 nChannels = 4
79 pathOut = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/merged"
79 pathOut = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/merged"
80 p0 = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/d2022240_Ch0"
80 p0 = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/d2022240_Ch0"
81 p1 = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/d2022240_Ch1"
81 p1 = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/d2022240_Ch1"
82 p2 = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/d2022240_Ch2"
82 p2 = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/d2022240_Ch2"
83 p3 = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/d2022240_Ch3"
83 p3 = "/home/soporte/Data/OutTest/clean2D/perpAndObliq/byChannels/d2022240_Ch3"
84 list = ['data_spc','data_cspc','nIncohInt','utctime']
84 list = ['data_spc','data_cspc','nIncohInt','utctime']
85 merger = MergeH5(nChannels,pathOut,list, p0, p1,p2,p3)
85 merger = MergeH5(nChannels,pathOut,list, p0, p1,p2,p3)
86 merger.run()
86 merger.run()
87
87
88 """
88 """
89
89
90 # #__attrs__ = ['paths', 'nChannels']
90 # #__attrs__ = ['paths', 'nChannels']
91 isConfig = False
91 isConfig = False
92 inPaths = None
92 inPaths = None
93 nChannels = None
93 nChannels = None
94 ch_dataIn = []
94 ch_dataIn = []
95
95
96 channelList = []
96 channelList = []
97
97
98 def __init__(self,nChannels, pOut, dataList, *args):
98 def __init__(self,nChannels, pOut, dataList, *args):
99
99
100 self.inPaths = [p for p in args]
100 self.inPaths = [p for p in args]
101 #print(self.inPaths)
101 #print(self.inPaths)
102 if len(self.inPaths) != nChannels:
102 if len(self.inPaths) != nChannels:
103 print("ERROR, number of channels different from iput paths {} != {}".format(nChannels, len(args)))
103 print("ERROR, number of channels different from iput paths {} != {}".format(nChannels, len(args)))
104 return
104 return
105
105
106 self.pathOut = pOut
106 self.pathOut = pOut
107 self.dataList = dataList
107 self.dataList = dataList
108 self.nChannels = len(self.inPaths)
108 self.nChannels = len(self.inPaths)
109 self.ch_dataIn = [Parameters() for p in args]
109 self.ch_dataIn = [Parameters() for p in args]
110 self.dataOut = Parameters()
110 self.dataOut = Parameters()
111 self.channelList = [n for n in range(nChannels)]
111 self.channelList = [n for n in range(nChannels)]
112 self.blocksPerFile = None
112 self.blocksPerFile = None
113 self.date = None
113 self.date = None
114 self.ext = ".hdf5$"
114 self.ext = ".hdf5$"
115 self.dataList = dataList
115 self.dataList = dataList
116 self.optchar = "D"
116 self.optchar = "D"
117 self.meta = {}
117 self.meta = {}
118 self.data = {}
118 self.data = {}
119 self.open_file = h5py.File
119 self.open_file = h5py.File
120 self.open_mode = 'r'
120 self.open_mode = 'r'
121 self.description = {}
121 self.description = {}
122 self.extras = {}
122 self.extras = {}
123 self.filefmt = "*%Y%j***"
123 self.filefmt = "*%Y%j***"
124 self.folderfmt = "*%Y%j"
124 self.folderfmt = "*%Y%j"
125
125
126 self.flag_spc = False
127 self.flag_pow = False
128 self.flag_snr = False
129 self.flag_nIcoh = False
130 self.flagProcessingHeader = False
131 self.flagControllerHeader = False
126
132
127 def setup(self):
133 def setup(self):
128
134
129
135
130 # if not self.ext.startswith('.'):
136 # if not self.ext.startswith('.'):
131 # self.ext = '.{}'.format(self.ext)
137 # self.ext = '.{}'.format(self.ext)
132
138
133 self.filenameList = self.searchFiles(self.inPaths, None)
139 self.filenameList = self.searchFiles(self.inPaths, None)
134 self.nfiles = len(self.filenameList[0])
140 self.nfiles = len(self.filenameList[0])
135
141
136
142
137
143
138 def searchFiles(self, paths, date, walk=True):
144 def searchFiles(self, paths, date, walk=True):
139 # self.paths = path
145 # self.paths = path
140 #self.date = startDate
146 #self.date = startDate
141 #self.walk = walk
147 #self.walk = walk
142 filenameList = [[] for n in range(self.nChannels)]
148 filenameList = [[] for n in range(self.nChannels)]
143 ch = 0
149 ch = 0
144 for path in paths:
150 for path in paths:
145 if os.path.exists(path):
151 if os.path.exists(path):
146 print("Searching files in {}".format(path))
152 print("Searching files in {}".format(path))
147 filenameList[ch] = self.getH5files(path, walk)
153 filenameList[ch] = self.getH5files(path, walk)
148 print("Found: ")
154 print("Found: ")
149 for f in filenameList[ch]:
155 for f in filenameList[ch]:
150 print(f)
156 print(f)
151 else:
157 else:
152 self.status = 0
158 self.status = 0
153 print('Path:%s does not exists'%path)
159 print('Path:%s does not exists'%path)
154 return 0
160 return 0
155 ch+=1
161 ch+=1
156 return filenameList
162 return filenameList
157
163
158 def getH5files(self, path, walk):
164 def getH5files(self, path, walk):
159
165
160 dirnameList = []
166 dirnameList = []
161 pat = '(\d)+.'+self.ext
167 pat = '(\d)+.'+self.ext
162 if walk:
168 if walk:
163 for root, dirs, files in os.walk(path):
169 for root, dirs, files in os.walk(path):
164 for dir in dirs:
170 for dir in dirs:
165 #print(os.path.join(root,dir))
171 #print(os.path.join(root,dir))
166 files = [re.search(pat,x) for x in os.listdir(os.path.join(root,dir))]
172 files = [re.search(pat,x) for x in os.listdir(os.path.join(root,dir))]
167 #print(files)
173 #print(files)
168 files = [x for x in files if x!=None]
174 files = [x for x in files if x!=None]
169 files = [x.string for x in files]
175 files = [x.string for x in files]
170 files = [os.path.join(root,dir,x) for x in files if x!=None]
176 files = [os.path.join(root,dir,x) for x in files if x!=None]
171 files.sort()
177 files.sort()
172
178
173 dirnameList += files
179 dirnameList += files
174 return dirnameList
180 return dirnameList
175 else:
181 else:
176
182
177 dirnameList = [re.search(pat,x) for x in os.listdir(path)]
183 dirnameList = [re.search(pat,x) for x in os.listdir(path)]
178 dirnameList = [x for x in dirnameList if x!=None]
184 dirnameList = [x for x in dirnameList if x!=None]
179 dirnameList = [x.string for x in dirnameList]
185 dirnameList = [x.string for x in dirnameList]
180 dirnameList = [x for x in dirnameList if x!=None]
186 dirnameList = [x for x in dirnameList if x!=None]
181 dirnameList.sort()
187 dirnameList.sort()
182
188
183 return dirnameList
189 return dirnameList
184
190
185
191
186 def readFile(self,fp,ch):
192 def readFile(self,fp,ch):
187
193
188 '''Read metadata and data'''
194 '''Read metadata and data'''
189
195
190 self.readMetadata(fp)
196 self.readMetadata(fp,ch)
197 #print(self.metadataList)
191 data = self.readData(fp)
198 data = self.readData(fp)
192
199
193
200
194 for attr in self.meta:
201 for attr in self.meta:
195 setattr(self.ch_dataIn[ch], attr, self.meta[attr])
202
203 if "processingHeaderObj" in attr:
204 self.flagProcessingHeader=True
205
206 if "radarControllerHeaderObj" in attr:
207 self.flagControllerHeader=True
208
209 at = attr.split('.')
210 #print("AT ", at)
211 if len(at) > 1:
212 setattr(eval("self.ch_dataIn[ch]."+at[0]),at[1], self.meta[attr])
213 else:
214 setattr(self.ch_dataIn[ch], attr, self.meta[attr])
196
215
197 self.fill_dataIn(data, self.ch_dataIn[ch])
216 self.fill_dataIn(data, self.ch_dataIn[ch])
198
217
199
218
200 return
219 return
201
220
202
221
203 def readMetadata(self, fp):
222 def readMetadata(self, fp, ch):
204 '''
223 '''
205 Reads Metadata
224 Reads Metadata
206 '''
225 '''
207 meta = {}
226 meta = {}
208 self.metadataList = []
227 self.metadataList = []
209 grp = fp['Metadata']
228 grp = fp['Metadata']
210 for name in grp:
229 for item in grp.values():
211 meta[name] = grp[name][()]
230 name = item.name
212 self.metadataList.append(name)
231
232 if isinstance(item, h5py.Dataset):
233 name = name.split("/")[-1]
234 if 'List' in name:
235 meta[name] = item[()].tolist()
236 else:
237 meta[name] = item[()]
238 self.metadataList.append(name)
239 else:
240 grp2 = fp[name]
241 Obj = name.split("/")[-1]
242 #print(Obj)
243 for item2 in grp2.values():
244 name2 = Obj+"."+item2.name.split("/")[-1]
245 if 'List' in name2:
246 meta[name2] = item2[()].tolist()
247 else:
248 meta[name2] = item2[()]
249 self.metadataList.append(name2)
250
251
213
252
214 for k in meta:
215 if ('List' in k):
216 meta[k] = meta[k].tolist()
217 if not self.meta:
253 if not self.meta:
218 self.meta = meta
254 self.meta = meta.copy()
219 self.meta["channelList"] =[n for n in range(self.nChannels)]
255 for key in list(self.meta.keys()):
256 if "channelList" in key:
257 self.meta["channelList"] =[n for n in range(self.nChannels)]
258 if "processingHeaderObj" in key:
259 self.meta["processingHeaderObj.channelList"] =[n for n in range(self.nChannels)]
260 if "radarControllerHeaderObj" in key:
261 self.meta["radarControllerHeaderObj.channelList"] =[n for n in range(self.nChannels)]
220 return 1
262 return 1
263
221 else:
264 else:
222 if len(self.meta) == len(meta):
223 for k in meta:
224 if 'List' in k and 'channel' not in k and "height" not in k:
225 self.meta[k] += meta[k]
226
265
227 return 1
266 for k in list(self.meta.keys()):
228 else:
267 if 'List' in k and 'channel' not in k and "height" not in k and "radarControllerHeaderObj" not in k:
229 return 0
268 self.meta[k] += meta[k]
269
270 #print("Metadata: ",self.meta)
271 return 1
272
230
273
231
274
232
275
233 def fill_dataIn(self,data, dataIn):
276 def fill_dataIn(self,data, dataIn):
234
277
235 for attr in data:
278 for attr in data:
236 if data[attr].ndim == 1:
279 if data[attr].ndim == 1:
237 setattr(dataIn, attr, data[attr][:])
280 setattr(dataIn, attr, data[attr][:])
238 else:
281 else:
239 setattr(dataIn, attr, numpy.squeeze(data[attr][:,:]))
282 setattr(dataIn, attr, numpy.squeeze(data[attr][:,:]))
240 print("shape in", dataIn.data_spc.shape, len(dataIn.data_spc))
283 #print("shape in", dataIn.data_spc.shape, len(dataIn.data_spc))
241 if dataIn.data_spc.ndim > 3:
284 if self.flag_spc:
242 dataIn.data_spc = dataIn.data_spc[0]
285 if dataIn.data_spc.ndim > 3:
243 #print("shape in", dataIn.data_spc.shape)
286 dataIn.data_spc = dataIn.data_spc[0]
287 #print("shape in", dataIn.data_spc.shape)
244
288
245
289
246
290
247 def getBlocksPerFile(self):
291 def getBlocksPerFile(self):
248 b = numpy.zeros(self.nChannels)
292 b = numpy.zeros(self.nChannels)
249 for i in range(self.nChannels):
293 for i in range(self.nChannels):
250 b[i] = self.ch_dataIn[i].data_spc.shape[0] #number of blocks
294 if self.flag_spc:
295 b[i] = self.ch_dataIn[i].data_spc.shape[0] #number of blocks
296 elif self.flag_pow:
297 b[i] = self.ch_dataIn[i].data_pow.shape[0] #number of blocks
298 elif self.flag_snr:
299 b[i] = self.ch_dataIn[i].data_snr.shape[0] #number of blocks
251
300
252 self.blocksPerFile = int(b.min())
301 self.blocksPerFile = int(b.min())
253 iresh_ch = numpy.where(b > self.blocksPerFile)[0]
302 iresh_ch = numpy.where(b > self.blocksPerFile)[0]
254 if len(iresh_ch) > 0:
303 if len(iresh_ch) > 0:
255 for ich in iresh_ch:
304 for ich in iresh_ch:
256 for i in range(len(self.dataList)):
305 for i in range(len(self.dataList)):
257 if hasattr(self.ch_dataIn[ich], self.dataList[i]):
306 if hasattr(self.ch_dataIn[ich], self.dataList[i]):
258 # print("reshaping ", self.dataList[i])
307 # print("reshaping ", self.dataList[i])
259 # print(getattr(self.ch_dataIn[ich], self.dataList[i]).shape)
308 # print(getattr(self.ch_dataIn[ich], self.dataList[i]).shape)
260 dataAux = getattr(self.ch_dataIn[ich], self.dataList[i])
309 dataAux = getattr(self.ch_dataIn[ich], self.dataList[i])
261 setattr(self.ch_dataIn[ich], self.dataList[i], None)
310 setattr(self.ch_dataIn[ich], self.dataList[i], None)
262 setattr(self.ch_dataIn[ich], self.dataList[i], dataAux[0:self.blocksPerFile])
311 setattr(self.ch_dataIn[ich], self.dataList[i], dataAux[0:self.blocksPerFile])
263 # print(getattr(self.ch_dataIn[ich], self.dataList[i]).shape)
312 # print(getattr(self.ch_dataIn[ich], self.dataList[i]).shape)
264 else:
313 else:
265 return
314 return
266
315
267
316
268 def getLabel(self, name, x=None):
317 def getLabel(self, name, x=None):
269
318
270 if x is None:
319 if x is None:
271 if 'Data' in self.description:
320 if 'Data' in self.description:
272 data = self.description['Data']
321 data = self.description['Data']
273 if 'Metadata' in self.description:
322 if 'Metadata' in self.description:
274 data.update(self.description['Metadata'])
323 data.update(self.description['Metadata'])
275 else:
324 else:
276 data = self.description
325 data = self.description
277 if name in data:
326 if name in data:
278 if isinstance(data[name], str):
327 if isinstance(data[name], str):
279 return data[name]
328 return data[name]
280 elif isinstance(data[name], list):
329 elif isinstance(data[name], list):
281 return None
330 return None
282 elif isinstance(data[name], dict):
331 elif isinstance(data[name], dict):
283 for key, value in data[name].items():
332 for key, value in data[name].items():
284 return key
333 return key
285 return name
334 return name
286 else:
335 else:
287 if 'Metadata' in self.description:
336 if 'Metadata' in self.description:
288 meta = self.description['Metadata']
337 meta = self.description['Metadata']
289 else:
338 else:
290 meta = self.description
339 meta = self.description
291 if name in meta:
340 if name in meta:
292 if isinstance(meta[name], list):
341 if isinstance(meta[name], list):
293 return meta[name][x]
342 return meta[name][x]
294 elif isinstance(meta[name], dict):
343 elif isinstance(meta[name], dict):
295 for key, value in meta[name].items():
344 for key, value in meta[name].items():
296 return value[x]
345 return value[x]
297
346
298 if 'cspc' in name:
347 if 'cspc' in name:
299 return 'pair{:02d}'.format(x)
348 return 'pair{:02d}'.format(x)
300 else:
349 else:
301 return 'channel{:02d}'.format(x)
350 return 'channel{:02d}'.format(x)
302
351
303 def readData(self, fp):
352 def readData(self, fp):
304 #print("read fp: ", fp)
353 #print("read fp: ", fp)
305 data = {}
354 data = {}
306
355
307 grp = fp['Data']
356 grp = fp['Data']
308
357
309 for name in grp:
358 for name in grp:
359 if "spc" in name:
360 self.flag_spc = True
361 if "pow" in name:
362 self.flag_pow = True
363 if "snr" in name:
364 self.flag_snr = True
310 if isinstance(grp[name], h5py.Dataset):
365 if isinstance(grp[name], h5py.Dataset):
311 array = grp[name][()]
366 array = grp[name][()]
312 elif isinstance(grp[name], h5py.Group):
367 elif isinstance(grp[name], h5py.Group):
313 array = []
368 array = []
314 for ch in grp[name]:
369 for ch in grp[name]:
315 array.append(grp[name][ch][()])
370 array.append(grp[name][ch][()])
316 array = numpy.array(array)
371 array = numpy.array(array)
317 else:
372 else:
318 print('Unknown type: {}'.format(name))
373 print('Unknown type: {}'.format(name))
319 data[name] = array
374 data[name] = array
320
375
321 return data
376 return data
322
377
323 def getDataOut(self):
378 def getDataOut(self):
324
379
325 self.dataOut = self.ch_dataIn[0].copy() #dataIn #blocks, fft, hei for metadata
380 self.dataOut = self.ch_dataIn[0].copy() #dataIn #blocks, fft, hei for metadata
326 if self.dataOut.data_spc.ndim < 3:
381 if self.flagProcessingHeader:
327 return 0
382 self.dataOut.processingHeaderObj = self.ch_dataIn[0].processingHeaderObj.copy()
383 self.dataOut.heightList = self.dataOut.processingHeaderObj.heightList
384 self.dataOut.ippSeconds = self.dataOut.processingHeaderObj.ipp
385 self.dataOut.channelList = self.dataOut.processingHeaderObj.channelList
386 self.dataOut.nCohInt = self.dataOut.processingHeaderObj.nCohInt
387 self.dataOut.nFFTPoints = self.dataOut.processingHeaderObj.nFFTPoints
388
389 if self.flagControllerHeader:
390 self.dataOut.radarControllerHeaderObj = self.ch_dataIn[0].radarControllerHeaderObj.copy()
391 self.dataOut.frequency = self.dataOut.radarControllerHeaderObj.frequency
392 #--------------------------------------------------------------------
393
394
395 #--------------------------------------------------------------------
396 if self.flag_spc:
397 if self.dataOut.data_spc.ndim < 3:
398 print("shape spc in: ",self.dataOut.data_spc.shape )
399 return 0
400 if self.flag_pow:
401 if self.dataOut.data_pow.ndim < 2:
402 print("shape pow in: ",self.dataOut.data_pow.shape )
403 return 0
404 if self.flag_snr:
405 if self.dataOut.data_snr.ndim < 2:
406 print("shape snr in: ",self.dataOut.data_snr.shape )
407 return 0
408
328 self.dataOut.data_spc = None
409 self.dataOut.data_spc = None
410 self.dataOut.data_cspc = None
411 self.dataOut.data_pow = None
412 self.dataOut.data_snr = None
329 self.dataOut.utctim = None
413 self.dataOut.utctim = None
330 self.dataOut.nIncohInt = None
414 self.dataOut.nIncohInt = None
331 #print(self.ch_dataIn[0].data_spc.shape)
415 #--------------------------------------------------------------------
332 spc = [data.data_spc for data in self.ch_dataIn]
416 if self.flag_spc:
333 self.dataOut.data_spc = numpy.stack(spc, axis=1) #blocks, ch, fft, hei
417 spc = [data.data_spc for data in self.ch_dataIn]
418 self.dataOut.data_spc = numpy.stack(spc, axis=1) #blocks, ch, fft, hei
419 #--------------------------------------------------------------------
420 if self.flag_pow:
421 pow = [data.data_pow for data in self.ch_dataIn]
422 self.dataOut.data_pow = numpy.stack(pow, axis=1) #blocks, ch, fft, hei
423 #--------------------------------------------------------------------
424 if self.flag_snr:
425 snr = [data.data_snr for data in self.ch_dataIn]
426 self.dataOut.data_snr = numpy.stack(snr, axis=1) #blocks, ch, fft, hei
427
428 #--------------------------------------------------------------------
334 time = [data.utctime for data in self.ch_dataIn]
429 time = [data.utctime for data in self.ch_dataIn]
335 time = numpy.asarray(time).mean(axis=0)
430 time = numpy.asarray(time).mean(axis=0)
336 #time = numpy.reshape(time, (len(time),1))
337 time = numpy.squeeze(time)
431 time = numpy.squeeze(time)
338 self.dataOut.utctime = time
432 self.dataOut.utctime = time
339 ints = [data.nIncohInt for data in self.ch_dataIn]
433 #--------------------------------------------------------------------
340 self.dataOut.nIncohInt = numpy.stack(ints, axis=1)
434 if self.flag_nIcoh:
341
435 ints = [data.nIncohInt for data in self.ch_dataIn]
342
436 self.dataOut.nIncohInt = numpy.stack(ints, axis=1)
343 if self.dataOut.nIncohInt.ndim > 3:
437
344 aux = self.dataOut.nIncohInt
438 if self.dataOut.nIncohInt.ndim > 3:
345 self.dataOut.nIncohInt = None
439 aux = self.dataOut.nIncohInt
346 self.dataOut.nIncohInt = aux[0]
440 self.dataOut.nIncohInt = None
347
441 self.dataOut.nIncohInt = aux[0]
442
443 if self.dataOut.nIncohInt.ndim < 3:
444 nIncohInt = numpy.repeat(self.dataOut.nIncohInt, self.dataOut.nHeights).reshape(self.blocksPerFile,self.nChannels, self.dataOut.nHeights)
445 #nIncohInt = numpy.reshape(nIncohInt, (self.blocksPerFile,self.nChannels, self.dataOut.nHeights))
446 self.dataOut.nIncohInt = None
447 self.dataOut.nIncohInt = nIncohInt
448
449 if (self.dataOut.nIncohInt.shape)[0]==self.nChannels: ## ch,blocks, hei
450 self.dataOut.nIncohInt = numpy.swapaxes(self.dataOut.nIncohInt, 0, 1) ## blocks,ch, hei
451 #--------------------------------------------------------------------
452 #print("utcTime: ", time.shape)
453 #print("data_spc ",self.dataOut.data_spc.shape)
454 if "data_cspc" in self.dataList:
455 pairsList = [pair for pair in itertools.combinations(self.channelList, 2)]
456 #print("PairsList: ", pairsList)
457 self.dataOut.pairsList = pairsList
458 cspc = []
348
459
349 if self.dataOut.nIncohInt.ndim < 3:
460 for i, j in pairsList:
350 nIncohInt = numpy.repeat(self.dataOut.nIncohInt, self.dataOut.nHeights).reshape(self.blocksPerFile,self.nChannels, self.dataOut.nHeights)
461 cspc.append(self.ch_dataIn[i].data_spc*numpy.conjugate(self.ch_dataIn[j].data_spc)) #blocks, fft, hei
351 #nIncohInt = numpy.reshape(nIncohInt, (self.blocksPerFile,self.nChannels, self.dataOut.nHeights))
352 self.dataOut.nIncohInt = None
353 self.dataOut.nIncohInt = nIncohInt
354
462
355 if (self.dataOut.nIncohInt.shape)[0]==self.nChannels: ## ch,blocks, hei
463 cspc = numpy.asarray(cspc) # # pairs, blocks, fft, hei
356 self.dataOut.nIncohInt = numpy.swapaxes(self.dataOut.nIncohInt, 0, 1) ## blocks,ch, hei
464 #print("cspc: ",cspc.shape)
465 self.dataOut.data_cspc = numpy.swapaxes(cspc, 0, 1) ## blocks, pairs, fft, hei
466 #print("dataOut.data_cspc: ",self.dataOut.data_cspc.shape)
467 #if "data_pow" in self.dataList:
357
468
358 #print("utcTime: ", time.shape)
359 #print("data_spc ",self.dataOut.data_spc.shape)
360 pairsList = [pair for pair in itertools.combinations(self.channelList, 2)]
361 #print("PairsList: ", pairsList)
362 self.dataOut.pairsList = pairsList
363 cspc = []
364
365 for i, j in pairsList:
366 cspc.append(self.ch_dataIn[i].data_spc*numpy.conjugate(self.ch_dataIn[j].data_spc)) #blocks, fft, hei
367
368 cspc = numpy.asarray(cspc) # # pairs, blocks, fft, hei
369 #print("cspc: ",cspc.shape)
370 self.dataOut.data_cspc = numpy.swapaxes(cspc, 0, 1) ## blocks, pairs, fft, hei
371 #print("dataOut.data_cspc: ",self.dataOut.data_cspc.shape)
372 return 1
469 return 1
373
470
471 # def writeMetadata(self, fp):
472 #
473 #
474 # grp = fp.create_group('Metadata')
475 #
476 # for i in range(len(self.metadataList)):
477 # if not hasattr(self.dataOut, self.metadataList[i]):
478 # print('Metadata: `{}` not found'.format(self.metadataList[i]))
479 # continue
480 # value = getattr(self.dataOut, self.metadataList[i])
481 # if isinstance(value, bool):
482 # if value is True:
483 # value = 1
484 # else:
485 # value = 0
486 # grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
487 # return
374 def writeMetadata(self, fp):
488 def writeMetadata(self, fp):
375
489
376
490
377 grp = fp.create_group('Metadata')
491 grp = fp.create_group('Metadata')
378
492
379 for i in range(len(self.metadataList)):
493 for i in range(len(self.metadataList)):
380 if not hasattr(self.dataOut, self.metadataList[i]):
494 attribute = self.metadataList[i]
381 print('Metadata: `{}` not found'.format(self.metadataList[i]))
495 attr = attribute.split('.')
382 continue
496 if '' in attr:
383 value = getattr(self.dataOut, self.metadataList[i])
497 attr.remove('')
384 if isinstance(value, bool):
498 #print(attr)
385 if value is True:
499 if len(attr) > 1:
386 value = 1
500 if not hasattr(eval("self.dataOut."+attr[0]),attr[1]):
501 print('Metadata: {}.{} not found'.format(attr[0],attr[1]))
502 continue
503 value = getattr(eval("self.dataOut."+attr[0]),attr[1])
504 if isinstance(value, bool):
505 if value is True:
506 value = 1
507 else:
508 value = 0
509 grp2 = None
510 if not 'Metadata/'+attr[0] in fp:
511 grp2 = fp.create_group('Metadata/'+attr[0])
387 else:
512 else:
388 value = 0
513 grp2 = fp['Metadata/'+attr[0]]
389 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
514
515 grp2.create_dataset(attr[1], data=value)
516
517 else:
518 if not hasattr(self.dataOut, attr[0] ):
519 print('Metadata: `{}` not found'.format(attribute))
520 continue
521 value = getattr(self.dataOut, attr[0])
522 if isinstance(value, bool):
523 if value is True:
524 value = 1
525 else:
526 value = 0
527 grp.create_dataset(self.getLabel(attribute), data=value)
390 return
528 return
391
529
392 def getDsList(self):
530 def getDsList(self):
393
531
394 dsList =[]
532 dsList =[]
395 for i in range(len(self.dataList)):
533 for i in range(len(self.dataList)):
396 dsDict = {}
534 dsDict = {}
397 if hasattr(self.dataOut, self.dataList[i]):
535 if hasattr(self.dataOut, self.dataList[i]):
398 dataAux = getattr(self.dataOut, self.dataList[i])
536 dataAux = getattr(self.dataOut, self.dataList[i])
399 dsDict['variable'] = self.dataList[i]
537 dsDict['variable'] = self.dataList[i]
400 else:
538 else:
401 print('Attribute {} not found in dataOut'.format(self.dataList[i]))
539 print('Attribute {} not found in dataOut'.format(self.dataList[i]))
402 continue
540 continue
403
541
404 if dataAux is None:
542 if dataAux is None:
405 continue
543 continue
406 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
544 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
407 dsDict['nDim'] = 0
545 dsDict['nDim'] = 0
408 else:
546 else:
409
547
410 dsDict['nDim'] = len(dataAux.shape) -1
548 dsDict['nDim'] = len(dataAux.shape) -1
411 dsDict['shape'] = dataAux.shape
549 dsDict['shape'] = dataAux.shape
412
550
413 if len(dsDict['shape'])>=2:
551 if len(dsDict['shape'])>=2:
414 dsDict['dsNumber'] = dataAux.shape[1]
552 dsDict['dsNumber'] = dataAux.shape[1]
415 else:
553 else:
416 dsDict['dsNumber'] = 1
554 dsDict['dsNumber'] = 1
417 dsDict['dtype'] = dataAux.dtype
555 dsDict['dtype'] = dataAux.dtype
418 # if len(dataAux.shape) == 4:
556 # if len(dataAux.shape) == 4:
419 # dsDict['nDim'] = len(dataAux.shape) -1
557 # dsDict['nDim'] = len(dataAux.shape) -1
420 # dsDict['shape'] = dataAux.shape
558 # dsDict['shape'] = dataAux.shape
421 # dsDict['dsNumber'] = dataAux.shape[1]
559 # dsDict['dsNumber'] = dataAux.shape[1]
422 # dsDict['dtype'] = dataAux.dtype
560 # dsDict['dtype'] = dataAux.dtype
423 # else:
561 # else:
424 # dsDict['nDim'] = len(dataAux.shape)
562 # dsDict['nDim'] = len(dataAux.shape)
425 # dsDict['shape'] = dataAux.shape
563 # dsDict['shape'] = dataAux.shape
426 # dsDict['dsNumber'] = dataAux.shape[0]
564 # dsDict['dsNumber'] = dataAux.shape[0]
427 # dsDict['dtype'] = dataAux.dtype
565 # dsDict['dtype'] = dataAux.dtype
428
566
429 dsList.append(dsDict)
567 dsList.append(dsDict)
430 #print(dsList)
568 #print(dsList)
431 self.dsList = dsList
569 self.dsList = dsList
432
570
433 def clean_dataIn(self):
571 def clean_dataIn(self):
434 for ch in range(self.nChannels):
572 for ch in range(self.nChannels):
435 self.ch_dataIn[ch].data_spc = None
573 self.ch_dataIn[ch].data_spc = None
436 self.ch_dataIn[ch].utctime = None
574 self.ch_dataIn[ch].utctime = None
437 self.ch_dataIn[ch].nIncohInt = None
575 self.ch_dataIn[ch].nIncohInt = None
438 self.meta ={}
576 self.meta ={}
439 self.blocksPerFile = None
577 self.blocksPerFile = None
440
578
441 def writeData(self, outFilename):
579 def writeData(self, outFilename):
442
580
443 self.getDsList()
581 self.getDsList()
444
582
445 fp = h5py.File(outFilename, 'w')
583 fp = h5py.File(outFilename, 'w')
446 self.writeMetadata(fp)
584 self.writeMetadata(fp)
447 grp = fp.create_group('Data')
585 grp = fp.create_group('Data')
448
586
449 dtsets = []
587 dtsets = []
450 data = []
588 data = []
451 for dsInfo in self.dsList:
589 for dsInfo in self.dsList:
452 if dsInfo['nDim'] == 0:
590 if dsInfo['nDim'] == 0:
453 ds = grp.create_dataset(
591 ds = grp.create_dataset(
454 self.getLabel(dsInfo['variable']),(self.blocksPerFile, ),chunks=True,dtype=numpy.float64)
592 self.getLabel(dsInfo['variable']),(self.blocksPerFile, ),chunks=True,dtype=numpy.float64)
455 dtsets.append(ds)
593 dtsets.append(ds)
456 data.append((dsInfo['variable'], -1))
594 data.append((dsInfo['variable'], -1))
457 else:
595 else:
458 label = self.getLabel(dsInfo['variable'])
596 label = self.getLabel(dsInfo['variable'])
459 if label is not None:
597 if label is not None:
460 sgrp = grp.create_group(label)
598 sgrp = grp.create_group(label)
461 else:
599 else:
462 sgrp = grp
600 sgrp = grp
463 k = -1*(dsInfo['nDim'] - 1)
601 k = -1*(dsInfo['nDim'] - 1)
464 #print(k, dsInfo['shape'], dsInfo['shape'][k:])
602 #print(k, dsInfo['shape'], dsInfo['shape'][k:])
465 for i in range(dsInfo['dsNumber']):
603 for i in range(dsInfo['dsNumber']):
466 ds = sgrp.create_dataset(
604 ds = sgrp.create_dataset(
467 self.getLabel(dsInfo['variable'], i),(self.blocksPerFile, ) + dsInfo['shape'][k:],
605 self.getLabel(dsInfo['variable'], i),(self.blocksPerFile, ) + dsInfo['shape'][k:],
468 chunks=True,
606 chunks=True,
469 dtype=dsInfo['dtype'])
607 dtype=dsInfo['dtype'])
470 dtsets.append(ds)
608 dtsets.append(ds)
471 data.append((dsInfo['variable'], i))
609 data.append((dsInfo['variable'], i))
472
610
473 #print("\n",dtsets)
611 #print("\n",dtsets)
474
612
475 print('Creating merged file: {}'.format(fp.filename))
613 print('Creating merged file: {}'.format(fp.filename))
476
614
477 for i, ds in enumerate(dtsets):
615 for i, ds in enumerate(dtsets):
478 attr, ch = data[i]
616 attr, ch = data[i]
479 if ch == -1:
617 if ch == -1:
480 ds[:] = getattr(self.dataOut, attr)
618 ds[:] = getattr(self.dataOut, attr)
481 else:
619 else:
482 #print(ds, getattr(self.dataOut, attr)[ch].shape)
620 #print(ds, getattr(self.dataOut, attr)[ch].shape)
483 aux = getattr(self.dataOut, attr)# block, ch, ...
621 aux = getattr(self.dataOut, attr)# block, ch, ...
484 aux = numpy.swapaxes(aux,0,1) # ch, blocks, ...
622 aux = numpy.swapaxes(aux,0,1) # ch, blocks, ...
485 #print(ds.shape, aux.shape)
623 #print(ds.shape, aux.shape)
486 #ds[:] = getattr(self.dataOut, attr)[ch]
624 #ds[:] = getattr(self.dataOut, attr)[ch]
487 ds[:] = aux[ch]
625 ds[:] = aux[ch]
488
626
489 fp.flush()
627 fp.flush()
490 fp.close()
628 fp.close()
491 self.clean_dataIn()
629 self.clean_dataIn()
492 return
630 return
493
631
494
632
495
633
496 def run(self):
634 def run(self):
497
635
498 if not(self.isConfig):
636 if not(self.isConfig):
499 self.setup()
637 self.setup()
500 self.isConfig = True
638 self.isConfig = True
501
639
502 for nf in range(self.nfiles):
640 for nf in range(self.nfiles):
503 name = None
641 name = None
504 for ch in range(self.nChannels):
642 for ch in range(self.nChannels):
505 name = self.filenameList[ch][nf]
643 name = self.filenameList[ch][nf]
506 filename = os.path.join(self.inPaths[ch], name)
644 filename = os.path.join(self.inPaths[ch], name)
507 fp = h5py.File(filename, 'r')
645 fp = h5py.File(filename, 'r')
508 #print("Opening file: ",filename)
646 #print("Opening file: ",filename)
509 self.readFile(fp,ch)
647 self.readFile(fp,ch)
510 fp.close()
648 fp.close()
511
649
512 if self.blocksPerFile == None:
650 if self.blocksPerFile == None:
513 self.getBlocksPerFile()
651 self.getBlocksPerFile()
514 print("blocks per file: ", self.blocksPerFile)
652 print("blocks per file: ", self.blocksPerFile)
515
653
516 if not self.getDataOut():
654 if not self.getDataOut():
517 print("Error getting DataOut invalid number of blocks")
655 print("Error getting DataOut invalid number of blocks")
518 return
656 return
519 name = name[-16:]
657 name = name[-16:]
520 #print("Final name out: ", name)
658 #print("Final name out: ", name)
521 outFile = os.path.join(self.pathOut, name)
659 outFile = os.path.join(self.pathOut, name)
522 #print("Outfile: ", outFile)
660 #print("Outfile: ", outFile)
523 self.writeData(outFile)
661 self.writeData(outFile)
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
@@ -1,2164 +1,2191
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """Spectra processing Unit and operations
5 """Spectra processing Unit and operations
6
6
7 Here you will find the processing unit `SpectraProc` and several operations
7 Here you will find the processing unit `SpectraProc` and several operations
8 to work with Spectra data type
8 to work with Spectra data type
9 """
9 """
10
10
11 import time
11 import time
12 import itertools
12 import itertools
13
13
14 import numpy
14 import numpy
15 import math
15 import math
16
16
17 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
17 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
18 from schainpy.model.data.jrodata import Spectra
18 from schainpy.model.data.jrodata import Spectra
19 from schainpy.model.data.jrodata import hildebrand_sekhon
19 from schainpy.model.data.jrodata import hildebrand_sekhon
20 from schainpy.model.data import _noise
20 from schainpy.model.data import _noise
21
21
22 from schainpy.utils import log
22 from schainpy.utils import log
23 import matplotlib.pyplot as plt
23 import matplotlib.pyplot as plt
24 #from scipy.optimize import curve_fit
24 #from scipy.optimize import curve_fit
25 from schainpy.model.io.utilsIO import getHei_index
25 from schainpy.model.io.utilsIO import getHei_index
26
26
27 class SpectraProc(ProcessingUnit):
27 class SpectraProc(ProcessingUnit):
28
28
29 def __init__(self):
29 def __init__(self):
30
30
31 ProcessingUnit.__init__(self)
31 ProcessingUnit.__init__(self)
32
32
33 self.buffer = None
33 self.buffer = None
34 self.firstdatatime = None
34 self.firstdatatime = None
35 self.profIndex = 0
35 self.profIndex = 0
36 self.dataOut = Spectra()
36 self.dataOut = Spectra()
37 self.id_min = None
37 self.id_min = None
38 self.id_max = None
38 self.id_max = None
39 self.setupReq = False #Agregar a todas las unidades de proc
39 self.setupReq = False #Agregar a todas las unidades de proc
40 self.nsamplesFFT = 0
40
41
41 def __updateSpecFromVoltage(self):
42 def __updateSpecFromVoltage(self):
42
43
43
44
44
45
45 self.dataOut.timeZone = self.dataIn.timeZone
46 self.dataOut.timeZone = self.dataIn.timeZone
46 self.dataOut.dstFlag = self.dataIn.dstFlag
47 self.dataOut.dstFlag = self.dataIn.dstFlag
47 self.dataOut.errorCount = self.dataIn.errorCount
48 self.dataOut.errorCount = self.dataIn.errorCount
48 self.dataOut.useLocalTime = self.dataIn.useLocalTime
49 self.dataOut.useLocalTime = self.dataIn.useLocalTime
49 try:
50
50 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
51 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
51 except:
52 pass
53 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
52 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
53 self.dataOut.ippSeconds = self.dataIn.ippSeconds
54 self.dataOut.ipp = self.dataIn.ipp
54 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
55 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
55 self.dataOut.channelList = self.dataIn.channelList
56 self.dataOut.channelList = self.dataIn.channelList
56 self.dataOut.heightList = self.dataIn.heightList
57 self.dataOut.heightList = self.dataIn.heightList
57 self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
58 self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
58 self.dataOut.nProfiles = self.dataOut.nFFTPoints
59 self.dataOut.nProfiles = self.dataOut.nFFTPoints
59 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
60 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
60 self.dataOut.utctime = self.firstdatatime
61 self.dataOut.utctime = self.firstdatatime
61 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
62 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
62 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
63 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
63 self.dataOut.flagShiftFFT = False
64 self.dataOut.flagShiftFFT = False
64 self.dataOut.nCohInt = self.dataIn.nCohInt
65 self.dataOut.nCohInt = self.dataIn.nCohInt
65 self.dataOut.nIncohInt = 1
66 self.dataOut.nIncohInt = 1
66 self.dataOut.radar_ipp = self.dataIn.radar_ipp
67 self.dataOut.sampled_heightsFFT = self.dataIn.sampled_heightsFFT
68 self.dataOut.pulseLength_TxA = self.dataIn.pulseLength_TxA
69 self.dataOut.deltaHeight = self.dataIn.deltaHeight
67 self.dataOut.deltaHeight = self.dataIn.deltaHeight
70 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
68 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
71 self.dataOut.frequency = self.dataIn.frequency
69 self.dataOut.frequency = self.dataIn.frequency
72 self.dataOut.realtime = self.dataIn.realtime
70 self.dataOut.realtime = self.dataIn.realtime
73 self.dataOut.azimuth = self.dataIn.azimuth
71 self.dataOut.azimuth = self.dataIn.azimuth
74 self.dataOut.zenith = self.dataIn.zenith
72 self.dataOut.zenith = self.dataIn.zenith
75 self.dataOut.codeList = self.dataIn.codeList
73 self.dataOut.codeList = self.dataIn.codeList
76 self.dataOut.azimuthList = self.dataIn.azimuthList
74 self.dataOut.azimuthList = self.dataIn.azimuthList
77 self.dataOut.elevationList = self.dataIn.elevationList
75 self.dataOut.elevationList = self.dataIn.elevationList
78
76
79
77
80 def __getFft(self):
78 def __getFft(self):
81 # print("fft donw")
79 # print("fft donw")
82 """
80 """
83 Convierte valores de Voltaje a Spectra
81 Convierte valores de Voltaje a Spectra
84
82
85 Affected:
83 Affected:
86 self.dataOut.data_spc
84 self.dataOut.data_spc
87 self.dataOut.data_cspc
85 self.dataOut.data_cspc
88 self.dataOut.data_dc
86 self.dataOut.data_dc
89 self.dataOut.heightList
87 self.dataOut.heightList
90 self.profIndex
88 self.profIndex
91 self.buffer
89 self.buffer
92 self.dataOut.flagNoData
90 self.dataOut.flagNoData
93 """
91 """
94 fft_volt = numpy.fft.fft(
92 fft_volt = numpy.fft.fft(
95 self.buffer, n=self.dataOut.nFFTPoints, axis=1)
93 self.buffer, n=self.dataOut.nFFTPoints, axis=1)
96 fft_volt = fft_volt.astype(numpy.dtype('complex'))
94 fft_volt = fft_volt.astype(numpy.dtype('complex'))
97 dc = fft_volt[:, 0, :]
95 dc = fft_volt[:, 0, :]
98
96
99 # calculo de self-spectra
97 # calculo de self-spectra
100 fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
98 fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
101 spc = fft_volt * numpy.conjugate(fft_volt)
99 spc = fft_volt * numpy.conjugate(fft_volt)
102 spc = spc.real
100 spc = spc.real
103
101
104 blocksize = 0
102 blocksize = 0
105 blocksize += dc.size
103 blocksize += dc.size
106 blocksize += spc.size
104 blocksize += spc.size
107
105
108 cspc = None
106 cspc = None
109 pairIndex = 0
107 pairIndex = 0
110 if self.dataOut.pairsList != None:
108 if self.dataOut.pairsList != None:
111 # calculo de cross-spectra
109 # calculo de cross-spectra
112 cspc = numpy.zeros(
110 cspc = numpy.zeros(
113 (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
111 (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
114 for pair in self.dataOut.pairsList:
112 for pair in self.dataOut.pairsList:
115 if pair[0] not in self.dataOut.channelList:
113 if pair[0] not in self.dataOut.channelList:
116 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
114 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
117 str(pair), str(self.dataOut.channelList)))
115 str(pair), str(self.dataOut.channelList)))
118 if pair[1] not in self.dataOut.channelList:
116 if pair[1] not in self.dataOut.channelList:
119 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
117 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
120 str(pair), str(self.dataOut.channelList)))
118 str(pair), str(self.dataOut.channelList)))
121
119
122 cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
120 cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
123 numpy.conjugate(fft_volt[pair[1], :, :])
121 numpy.conjugate(fft_volt[pair[1], :, :])
124 pairIndex += 1
122 pairIndex += 1
125 blocksize += cspc.size
123 blocksize += cspc.size
126
124
127 self.dataOut.data_spc = spc
125 self.dataOut.data_spc = spc
128 self.dataOut.data_cspc = cspc
126 self.dataOut.data_cspc = cspc
129 self.dataOut.data_dc = dc
127 self.dataOut.data_dc = dc
130 self.dataOut.blockSize = blocksize
128 self.dataOut.blockSize = blocksize
131 self.dataOut.flagShiftFFT = False
129 self.dataOut.flagShiftFFT = False
132
130
133 def run(self, nProfiles=None, nFFTPoints=None, pairsList=None, ippFactor=None, shift_fft=False, zeroPad=False):
131 def run(self, nProfiles=None, nFFTPoints=None, pairsList=None, ippFactor=None, shift_fft=False, zeroPad=False):
134 #print("run spc proc")
132
135
133
136 try:
134 try:
137 type = self.dataIn.type.decode("utf-8")
135 type = self.dataIn.type.decode("utf-8")
138 self.dataIn.type = type
136 self.dataIn.type = type
139 except:
137 except:
140 pass
138 pass
141 if self.dataIn.type == "Spectra":
139 if self.dataIn.type == "Spectra":
142
140
143 try:
141 try:
144 self.dataOut.copy(self.dataIn)
142 self.dataOut.copy(self.dataIn)
143 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
144 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
145 self.dataOut.nProfiles = self.dataOut.nFFTPoints
145 self.dataOut.nProfiles = self.dataOut.nFFTPoints
146 #self.dataOut.nHeights = len(self.dataOut.heightList)
146 #self.dataOut.nHeights = len(self.dataOut.heightList)
147 except Exception as e:
147 except Exception as e:
148 print("Error dataIn ",e)
148 print("Error dataIn ",e)
149
149
150
151
150 if shift_fft:
152 if shift_fft:
151 #desplaza a la derecha en el eje 2 determinadas posiciones
153 #desplaza a la derecha en el eje 2 determinadas posiciones
152 shift = int(self.dataOut.nFFTPoints/2)
154 shift = int(self.dataOut.nFFTPoints/2)
153 self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
155 self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
154
156
155 if self.dataOut.data_cspc is not None:
157 if self.dataOut.data_cspc is not None:
156 #desplaza a la derecha en el eje 2 determinadas posiciones
158 #desplaza a la derecha en el eje 2 determinadas posiciones
157 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
159 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
158 if pairsList:
160 if pairsList:
159 self.__selectPairs(pairsList)
161 self.__selectPairs(pairsList)
160
162
161
163
162 elif self.dataIn.type == "Voltage":
164 elif self.dataIn.type == "Voltage":
163
165
164 self.dataOut.flagNoData = True
166 self.dataOut.flagNoData = True
165
167 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
168 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
166 if nFFTPoints == None:
169 if nFFTPoints == None:
167 raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
170 raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
168
171
169 if nProfiles == None:
172 if nProfiles == None:
170 nProfiles = nFFTPoints
173 nProfiles = nFFTPoints
171
174
172 if ippFactor == None:
175 if ippFactor == None:
173 self.dataOut.ippFactor = 1
176 self.dataOut.ippFactor = 1
174
177
175 self.dataOut.nFFTPoints = nFFTPoints
178 self.dataOut.nFFTPoints = nFFTPoints
176 #print(" volts ch,prof, h: ", self.dataIn.data.shape)
179 #print(" volts ch,prof, h: ", self.dataIn.data.shape)
177 if self.buffer is None:
180 if self.buffer is None:
178 if not zeroPad:
181 if not zeroPad:
179 self.buffer = numpy.zeros((self.dataIn.nChannels,
182 self.buffer = numpy.zeros((self.dataIn.nChannels,
180 nProfiles,
183 nProfiles,
181 self.dataIn.nHeights),
184 self.dataIn.nHeights),
182 dtype='complex')
185 dtype='complex')
183 else:
186 else:
184 self.buffer = numpy.zeros((self.dataIn.nChannels,
187 self.buffer = numpy.zeros((self.dataIn.nChannels,
185 nFFTPoints,
188 nFFTPoints,
186 self.dataIn.nHeights),
189 self.dataIn.nHeights),
187 dtype='complex')
190 dtype='complex')
188
191
189 if self.dataIn.flagDataAsBlock:
192 if self.dataIn.flagDataAsBlock:
190 nVoltProfiles = self.dataIn.data.shape[1]
193 nVoltProfiles = self.dataIn.data.shape[1]
191
194
192 if nVoltProfiles == nProfiles or zeroPad:
195 if nVoltProfiles == nProfiles or zeroPad:
193 self.buffer = self.dataIn.data.copy()
196 self.buffer = self.dataIn.data.copy()
194 self.profIndex = nVoltProfiles
197 self.profIndex = nVoltProfiles
195
198
196 elif nVoltProfiles < nProfiles:
199 elif nVoltProfiles < nProfiles:
197
200
198 if self.profIndex == 0:
201 if self.profIndex == 0:
199 self.id_min = 0
202 self.id_min = 0
200 self.id_max = nVoltProfiles
203 self.id_max = nVoltProfiles
201
204
202 self.buffer[:, self.id_min:self.id_max,
205 self.buffer[:, self.id_min:self.id_max,
203 :] = self.dataIn.data
206 :] = self.dataIn.data
204 self.profIndex += nVoltProfiles
207 self.profIndex += nVoltProfiles
205 self.id_min += nVoltProfiles
208 self.id_min += nVoltProfiles
206 self.id_max += nVoltProfiles
209 self.id_max += nVoltProfiles
207 else:
210 else:
208 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
211 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
209 self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
212 self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
210 self.dataOut.flagNoData = True
213 self.dataOut.flagNoData = True
211 else:
214 else:
212 self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
215 self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
213 self.profIndex += 1
216 self.profIndex += 1
214
217
215 if self.firstdatatime == None:
218 if self.firstdatatime == None:
216 self.firstdatatime = self.dataIn.utctime
219 self.firstdatatime = self.dataIn.utctime
217
220
218 if self.profIndex == nProfiles or zeroPad:
221 if self.profIndex == nProfiles or zeroPad:
219
222
220 self.__updateSpecFromVoltage()
223 self.__updateSpecFromVoltage()
221
224
222 if pairsList == None:
225 if pairsList == None:
223 self.dataOut.pairsList = [pair for pair in itertools.combinations(self.dataOut.channelList, 2)]
226 self.dataOut.pairsList = [pair for pair in itertools.combinations(self.dataOut.channelList, 2)]
224 else:
227 else:
225 self.dataOut.pairsList = pairsList
228 self.dataOut.pairsList = pairsList
226 self.__getFft()
229 self.__getFft()
227 self.dataOut.flagNoData = False
230 self.dataOut.flagNoData = False
228 self.firstdatatime = None
231 self.firstdatatime = None
232 self.nsamplesFFT = self.profIndex
229 self.profIndex = 0
233 self.profIndex = 0
230
234
235 #update Processing Header:
236 self.dataOut.processingHeaderObj.dtype = "Spectra"
237 self.dataOut.processingHeaderObj.nFFTPoints = self.dataOut.nFFTPoints
238 self.dataOut.processingHeaderObj.nSamplesFFT = self.nsamplesFFT
239 self.dataOut.processingHeaderObj.nIncohInt = 1
240
241
231 elif self.dataIn.type == "Parameters":
242 elif self.dataIn.type == "Parameters":
232
243
233 self.dataOut.data_spc = self.dataIn.data_spc
244 self.dataOut.data_spc = self.dataIn.data_spc
234 self.dataOut.data_cspc = self.dataIn.data_cspc
245 self.dataOut.data_cspc = self.dataIn.data_cspc
235 self.dataOut.data_outlier = self.dataIn.data_outlier
246 self.dataOut.data_outlier = self.dataIn.data_outlier
236 self.dataOut.nProfiles = self.dataIn.nProfiles
247 self.dataOut.nProfiles = self.dataIn.nProfiles
237 self.dataOut.nIncohInt = self.dataIn.nIncohInt
248 self.dataOut.nIncohInt = self.dataIn.nIncohInt
238 self.dataOut.nFFTPoints = self.dataIn.nFFTPoints
249 self.dataOut.nFFTPoints = self.dataIn.nFFTPoints
239 self.dataOut.ippFactor = self.dataIn.ippFactor
250 self.dataOut.ippFactor = self.dataIn.ippFactor
240 self.dataOut.max_nIncohInt = self.dataIn.max_nIncohInt
251 self.dataOut.max_nIncohInt = self.dataIn.max_nIncohInt
241 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
252 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
253 self.dataOut.ProcessingHeader = self.dataIn.ProcessingHeader.copy()
254 self.dataOut.ippSeconds = self.dataIn.ippSeconds
242 self.dataOut.ipp = self.dataIn.ipp
255 self.dataOut.ipp = self.dataIn.ipp
243 #self.dataOut.abscissaList = self.dataIn.getVelRange(1)
256 #self.dataOut.abscissaList = self.dataIn.getVelRange(1)
244 #self.dataOut.spc_noise = self.dataIn.getNoise()
257 #self.dataOut.spc_noise = self.dataIn.getNoise()
245 #self.dataOut.spc_range = (self.dataIn.getFreqRange(1) , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1))
258 #self.dataOut.spc_range = (self.dataIn.getFreqRange(1) , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1))
246 # self.dataOut.normFactor = self.dataIn.normFactor
259 # self.dataOut.normFactor = self.dataIn.normFactor
247 if hasattr(self.dataIn, 'channelList'):
260 if hasattr(self.dataIn, 'channelList'):
248 self.dataOut.channelList = self.dataIn.channelList
261 self.dataOut.channelList = self.dataIn.channelList
249 if hasattr(self.dataIn, 'pairsList'):
262 if hasattr(self.dataIn, 'pairsList'):
250 self.dataOut.pairsList = self.dataIn.pairsList
263 self.dataOut.pairsList = self.dataIn.pairsList
251 self.dataOut.groupList = self.dataIn.pairsList
264 self.dataOut.groupList = self.dataIn.pairsList
252
265
253 self.dataOut.flagNoData = False
266 self.dataOut.flagNoData = False
254
267
255 if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels
268 if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels
256 self.dataOut.ChanDist = self.dataIn.ChanDist
269 self.dataOut.ChanDist = self.dataIn.ChanDist
257 else: self.dataOut.ChanDist = None
270 else: self.dataOut.ChanDist = None
258
271
259 #if hasattr(self.dataIn, 'VelRange'): #Velocities range
272 #if hasattr(self.dataIn, 'VelRange'): #Velocities range
260 # self.dataOut.VelRange = self.dataIn.VelRange
273 # self.dataOut.VelRange = self.dataIn.VelRange
261 #else: self.dataOut.VelRange = None
274 #else: self.dataOut.VelRange = None
262
275
263
276
264
277
265 else:
278 else:
266 raise ValueError("The type of input object {} is not valid".format(
279 raise ValueError("The type of input object {} is not valid".format(
267 self.dataIn.type))
280 self.dataIn.type))
281
282
283
284
268 #print("spc proc Done", self.dataOut.data_spc.shape)
285 #print("spc proc Done", self.dataOut.data_spc.shape)
286 #print(self.dataOut.data_spc)
287 return
269
288
270 def __selectPairs(self, pairsList):
289 def __selectPairs(self, pairsList):
271
290
272 if not pairsList:
291 if not pairsList:
273 return
292 return
274
293
275 pairs = []
294 pairs = []
276 pairsIndex = []
295 pairsIndex = []
277
296
278 for pair in pairsList:
297 for pair in pairsList:
279 if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
298 if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
280 continue
299 continue
281 pairs.append(pair)
300 pairs.append(pair)
282 pairsIndex.append(pairs.index(pair))
301 pairsIndex.append(pairs.index(pair))
283
302
284 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
303 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
285 self.dataOut.pairsList = pairs
304 self.dataOut.pairsList = pairs
286
305
287 return
306 return
288
307
289 def selectFFTs(self, minFFT, maxFFT ):
308 def selectFFTs(self, minFFT, maxFFT ):
290 """
309 """
291 Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
310 Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
292 minFFT<= FFT <= maxFFT
311 minFFT<= FFT <= maxFFT
293 """
312 """
294
313
295 if (minFFT > maxFFT):
314 if (minFFT > maxFFT):
296 raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT))
315 raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT))
297
316
298 if (minFFT < self.dataOut.getFreqRange()[0]):
317 if (minFFT < self.dataOut.getFreqRange()[0]):
299 minFFT = self.dataOut.getFreqRange()[0]
318 minFFT = self.dataOut.getFreqRange()[0]
300
319
301 if (maxFFT > self.dataOut.getFreqRange()[-1]):
320 if (maxFFT > self.dataOut.getFreqRange()[-1]):
302 maxFFT = self.dataOut.getFreqRange()[-1]
321 maxFFT = self.dataOut.getFreqRange()[-1]
303
322
304 minIndex = 0
323 minIndex = 0
305 maxIndex = 0
324 maxIndex = 0
306 FFTs = self.dataOut.getFreqRange()
325 FFTs = self.dataOut.getFreqRange()
307
326
308 inda = numpy.where(FFTs >= minFFT)
327 inda = numpy.where(FFTs >= minFFT)
309 indb = numpy.where(FFTs <= maxFFT)
328 indb = numpy.where(FFTs <= maxFFT)
310
329
311 try:
330 try:
312 minIndex = inda[0][0]
331 minIndex = inda[0][0]
313 except:
332 except:
314 minIndex = 0
333 minIndex = 0
315
334
316 try:
335 try:
317 maxIndex = indb[0][-1]
336 maxIndex = indb[0][-1]
318 except:
337 except:
319 maxIndex = len(FFTs)
338 maxIndex = len(FFTs)
320
339
321 self.selectFFTsByIndex(minIndex, maxIndex)
340 self.selectFFTsByIndex(minIndex, maxIndex)
322
341
323 return 1
342 return 1
324
343
325 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
344 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
326 newheis = numpy.where(
345 newheis = numpy.where(
327 self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
346 self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
328
347
329 if hei_ref != None:
348 if hei_ref != None:
330 newheis = numpy.where(self.dataOut.heightList > hei_ref)
349 newheis = numpy.where(self.dataOut.heightList > hei_ref)
331
350
332 minIndex = min(newheis[0])
351 minIndex = min(newheis[0])
333 maxIndex = max(newheis[0])
352 maxIndex = max(newheis[0])
334 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
353 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
335 heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
354 heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
336
355
337 # determina indices
356 # determina indices
338 nheis = int(self.dataOut.radarControllerHeaderObj.txB /
357 nheis = int(self.dataOut.radarControllerHeaderObj.txB /
339 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
358 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
340 avg_dB = 10 * \
359 avg_dB = 10 * \
341 numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
360 numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
342 beacon_dB = numpy.sort(avg_dB)[-nheis:]
361 beacon_dB = numpy.sort(avg_dB)[-nheis:]
343 beacon_heiIndexList = []
362 beacon_heiIndexList = []
344 for val in avg_dB.tolist():
363 for val in avg_dB.tolist():
345 if val >= beacon_dB[0]:
364 if val >= beacon_dB[0]:
346 beacon_heiIndexList.append(avg_dB.tolist().index(val))
365 beacon_heiIndexList.append(avg_dB.tolist().index(val))
347
366
348 #data_spc = data_spc[:,:,beacon_heiIndexList]
367 #data_spc = data_spc[:,:,beacon_heiIndexList]
349 data_cspc = None
368 data_cspc = None
350 if self.dataOut.data_cspc is not None:
369 if self.dataOut.data_cspc is not None:
351 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
370 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
352 #data_cspc = data_cspc[:,:,beacon_heiIndexList]
371 #data_cspc = data_cspc[:,:,beacon_heiIndexList]
353
372
354 data_dc = None
373 data_dc = None
355 if self.dataOut.data_dc is not None:
374 if self.dataOut.data_dc is not None:
356 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
375 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
357 #data_dc = data_dc[:,beacon_heiIndexList]
376 #data_dc = data_dc[:,beacon_heiIndexList]
358
377
359 self.dataOut.data_spc = data_spc
378 self.dataOut.data_spc = data_spc
360 self.dataOut.data_cspc = data_cspc
379 self.dataOut.data_cspc = data_cspc
361 self.dataOut.data_dc = data_dc
380 self.dataOut.data_dc = data_dc
362 self.dataOut.heightList = heightList
381 self.dataOut.heightList = heightList
363 self.dataOut.beacon_heiIndexList = beacon_heiIndexList
382 self.dataOut.beacon_heiIndexList = beacon_heiIndexList
364
383
365 return 1
384 return 1
366
385
367 def selectFFTsByIndex(self, minIndex, maxIndex):
386 def selectFFTsByIndex(self, minIndex, maxIndex):
368 """
387 """
369
388
370 """
389 """
371
390
372 if (minIndex < 0) or (minIndex > maxIndex):
391 if (minIndex < 0) or (minIndex > maxIndex):
373 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (minIndex, maxIndex))
392 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (minIndex, maxIndex))
374
393
375 if (maxIndex >= self.dataOut.nProfiles):
394 if (maxIndex >= self.dataOut.nProfiles):
376 maxIndex = self.dataOut.nProfiles-1
395 maxIndex = self.dataOut.nProfiles-1
377
396
378 #Spectra
397 #Spectra
379 data_spc = self.dataOut.data_spc[:,minIndex:maxIndex+1,:]
398 data_spc = self.dataOut.data_spc[:,minIndex:maxIndex+1,:]
380
399
381 data_cspc = None
400 data_cspc = None
382 if self.dataOut.data_cspc is not None:
401 if self.dataOut.data_cspc is not None:
383 data_cspc = self.dataOut.data_cspc[:,minIndex:maxIndex+1,:]
402 data_cspc = self.dataOut.data_cspc[:,minIndex:maxIndex+1,:]
384
403
385 data_dc = None
404 data_dc = None
386 if self.dataOut.data_dc is not None:
405 if self.dataOut.data_dc is not None:
387 data_dc = self.dataOut.data_dc[minIndex:maxIndex+1,:]
406 data_dc = self.dataOut.data_dc[minIndex:maxIndex+1,:]
388
407
389 self.dataOut.data_spc = data_spc
408 self.dataOut.data_spc = data_spc
390 self.dataOut.data_cspc = data_cspc
409 self.dataOut.data_cspc = data_cspc
391 self.dataOut.data_dc = data_dc
410 self.dataOut.data_dc = data_dc
392
411
393 self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1])
412 self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1])
394 self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1]
413 self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1]
395 self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1]
414 self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1]
396
415
397 return 1
416 return 1
398
417
399 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
418 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
400 # validacion de rango
419 # validacion de rango
401 if minHei == None:
420 if minHei == None:
402 minHei = self.dataOut.heightList[0]
421 minHei = self.dataOut.heightList[0]
403
422
404 if maxHei == None:
423 if maxHei == None:
405 maxHei = self.dataOut.heightList[-1]
424 maxHei = self.dataOut.heightList[-1]
406
425
407 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
426 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
408 print('minHei: %.2f is out of the heights range' % (minHei))
427 print('minHei: %.2f is out of the heights range' % (minHei))
409 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
428 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
410 minHei = self.dataOut.heightList[0]
429 minHei = self.dataOut.heightList[0]
411
430
412 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
431 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
413 print('maxHei: %.2f is out of the heights range' % (maxHei))
432 print('maxHei: %.2f is out of the heights range' % (maxHei))
414 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
433 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
415 maxHei = self.dataOut.heightList[-1]
434 maxHei = self.dataOut.heightList[-1]
416
435
417 # validacion de velocidades
436 # validacion de velocidades
418 velrange = self.dataOut.getVelRange(1)
437 velrange = self.dataOut.getVelRange(1)
419
438
420 if minVel == None:
439 if minVel == None:
421 minVel = velrange[0]
440 minVel = velrange[0]
422
441
423 if maxVel == None:
442 if maxVel == None:
424 maxVel = velrange[-1]
443 maxVel = velrange[-1]
425
444
426 if (minVel < velrange[0]) or (minVel > maxVel):
445 if (minVel < velrange[0]) or (minVel > maxVel):
427 print('minVel: %.2f is out of the velocity range' % (minVel))
446 print('minVel: %.2f is out of the velocity range' % (minVel))
428 print('minVel is setting to %.2f' % (velrange[0]))
447 print('minVel is setting to %.2f' % (velrange[0]))
429 minVel = velrange[0]
448 minVel = velrange[0]
430
449
431 if (maxVel > velrange[-1]) or (maxVel < minVel):
450 if (maxVel > velrange[-1]) or (maxVel < minVel):
432 print('maxVel: %.2f is out of the velocity range' % (maxVel))
451 print('maxVel: %.2f is out of the velocity range' % (maxVel))
433 print('maxVel is setting to %.2f' % (velrange[-1]))
452 print('maxVel is setting to %.2f' % (velrange[-1]))
434 maxVel = velrange[-1]
453 maxVel = velrange[-1]
435
454
436 # seleccion de indices para rango
455 # seleccion de indices para rango
437 minIndex = 0
456 minIndex = 0
438 maxIndex = 0
457 maxIndex = 0
439 heights = self.dataOut.heightList
458 heights = self.dataOut.heightList
440
459
441 inda = numpy.where(heights >= minHei)
460 inda = numpy.where(heights >= minHei)
442 indb = numpy.where(heights <= maxHei)
461 indb = numpy.where(heights <= maxHei)
443
462
444 try:
463 try:
445 minIndex = inda[0][0]
464 minIndex = inda[0][0]
446 except:
465 except:
447 minIndex = 0
466 minIndex = 0
448
467
449 try:
468 try:
450 maxIndex = indb[0][-1]
469 maxIndex = indb[0][-1]
451 except:
470 except:
452 maxIndex = len(heights)
471 maxIndex = len(heights)
453
472
454 if (minIndex < 0) or (minIndex > maxIndex):
473 if (minIndex < 0) or (minIndex > maxIndex):
455 raise ValueError("some value in (%d,%d) is not valid" % (
474 raise ValueError("some value in (%d,%d) is not valid" % (
456 minIndex, maxIndex))
475 minIndex, maxIndex))
457
476
458 if (maxIndex >= self.dataOut.nHeights):
477 if (maxIndex >= self.dataOut.nHeights):
459 maxIndex = self.dataOut.nHeights - 1
478 maxIndex = self.dataOut.nHeights - 1
460
479
461 # seleccion de indices para velocidades
480 # seleccion de indices para velocidades
462 indminvel = numpy.where(velrange >= minVel)
481 indminvel = numpy.where(velrange >= minVel)
463 indmaxvel = numpy.where(velrange <= maxVel)
482 indmaxvel = numpy.where(velrange <= maxVel)
464 try:
483 try:
465 minIndexVel = indminvel[0][0]
484 minIndexVel = indminvel[0][0]
466 except:
485 except:
467 minIndexVel = 0
486 minIndexVel = 0
468
487
469 try:
488 try:
470 maxIndexVel = indmaxvel[0][-1]
489 maxIndexVel = indmaxvel[0][-1]
471 except:
490 except:
472 maxIndexVel = len(velrange)
491 maxIndexVel = len(velrange)
473
492
474 # seleccion del espectro
493 # seleccion del espectro
475 data_spc = self.dataOut.data_spc[:,
494 data_spc = self.dataOut.data_spc[:,
476 minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
495 minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
477 # estimacion de ruido
496 # estimacion de ruido
478 noise = numpy.zeros(self.dataOut.nChannels)
497 noise = numpy.zeros(self.dataOut.nChannels)
479
498
480 for channel in range(self.dataOut.nChannels):
499 for channel in range(self.dataOut.nChannels):
481 daux = data_spc[channel, :, :]
500 daux = data_spc[channel, :, :]
482 sortdata = numpy.sort(daux, axis=None)
501 sortdata = numpy.sort(daux, axis=None)
483 noise[channel] = hildebrand_sekhon(sortdata, self.dataOut.nIncohInt)
502 noise[channel] = hildebrand_sekhon(sortdata, self.dataOut.nIncohInt)
484
503
485 self.dataOut.noise_estimation = noise.copy()
504 self.dataOut.noise_estimation = noise.copy()
486
505
487 return 1
506 return 1
488
507
489 class removeDC(Operation):
508 class removeDC(Operation):
490
509
491 def run(self, dataOut, mode=2):
510 def run(self, dataOut, mode=2):
492 self.dataOut = dataOut
511 self.dataOut = dataOut
493 jspectra = self.dataOut.data_spc
512 jspectra = self.dataOut.data_spc
494 jcspectra = self.dataOut.data_cspc
513 jcspectra = self.dataOut.data_cspc
495
514
496 num_chan = jspectra.shape[0]
515 num_chan = jspectra.shape[0]
497 num_hei = jspectra.shape[2]
516 num_hei = jspectra.shape[2]
498
517
499 if jcspectra is not None:
518 if jcspectra is not None:
500 jcspectraExist = True
519 jcspectraExist = True
501 num_pairs = jcspectra.shape[0]
520 num_pairs = jcspectra.shape[0]
502 else:
521 else:
503 jcspectraExist = False
522 jcspectraExist = False
504
523
505 freq_dc = int(jspectra.shape[1] / 2)
524 freq_dc = int(jspectra.shape[1] / 2)
506 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
525 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
507 ind_vel = ind_vel.astype(int)
526 ind_vel = ind_vel.astype(int)
508
527
509 if ind_vel[0] < 0:
528 if ind_vel[0] < 0:
510 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
529 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
511
530
512 if mode == 1:
531 if mode == 1:
513 jspectra[:, freq_dc, :] = (
532 jspectra[:, freq_dc, :] = (
514 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
533 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
515
534
516 if jcspectraExist:
535 if jcspectraExist:
517 jcspectra[:, freq_dc, :] = (
536 jcspectra[:, freq_dc, :] = (
518 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
537 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
519
538
520 if mode == 2:
539 if mode == 2:
521
540
522 vel = numpy.array([-2, -1, 1, 2])
541 vel = numpy.array([-2, -1, 1, 2])
523 xx = numpy.zeros([4, 4])
542 xx = numpy.zeros([4, 4])
524
543
525 for fil in range(4):
544 for fil in range(4):
526 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
545 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
527
546
528 xx_inv = numpy.linalg.inv(xx)
547 xx_inv = numpy.linalg.inv(xx)
529 xx_aux = xx_inv[0, :]
548 xx_aux = xx_inv[0, :]
530
549
531 for ich in range(num_chan):
550 for ich in range(num_chan):
532 yy = jspectra[ich, ind_vel, :]
551 yy = jspectra[ich, ind_vel, :]
533 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
552 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
534
553
535 junkid = jspectra[ich, freq_dc, :] <= 0
554 junkid = jspectra[ich, freq_dc, :] <= 0
536 cjunkid = sum(junkid)
555 cjunkid = sum(junkid)
537
556
538 if cjunkid.any():
557 if cjunkid.any():
539 jspectra[ich, freq_dc, junkid.nonzero()] = (
558 jspectra[ich, freq_dc, junkid.nonzero()] = (
540 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
559 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
541
560
542 if jcspectraExist:
561 if jcspectraExist:
543 for ip in range(num_pairs):
562 for ip in range(num_pairs):
544 yy = jcspectra[ip, ind_vel, :]
563 yy = jcspectra[ip, ind_vel, :]
545 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
564 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
546
565
547 self.dataOut.data_spc = jspectra
566 self.dataOut.data_spc = jspectra
548 self.dataOut.data_cspc = jcspectra
567 self.dataOut.data_cspc = jcspectra
549
568
550 return self.dataOut
569 return self.dataOut
551
570
552 class getNoiseB(Operation):
571 class getNoiseB(Operation):
553
572
554 __slots__ =('offset','warnings', 'isConfig', 'minIndex','maxIndex','minIndexFFT','maxIndexFFT')
573 __slots__ =('offset','warnings', 'isConfig', 'minIndex','maxIndex','minIndexFFT','maxIndexFFT')
555 def __init__(self):
574 def __init__(self):
556
575
557 Operation.__init__(self)
576 Operation.__init__(self)
558 self.isConfig = False
577 self.isConfig = False
559
578
560 def setup(self, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
579 def setup(self, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
561
580
562 self.warnings = warnings
581 self.warnings = warnings
563 if minHei == None:
582 if minHei == None:
564 minHei = self.dataOut.heightList[0]
583 minHei = self.dataOut.heightList[0]
565
584
566 if maxHei == None:
585 if maxHei == None:
567 maxHei = self.dataOut.heightList[-1]
586 maxHei = self.dataOut.heightList[-1]
568
587
569 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
588 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
570 if self.warnings:
589 if self.warnings:
571 print('minHei: %.2f is out of the heights range' % (minHei))
590 print('minHei: %.2f is out of the heights range' % (minHei))
572 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
591 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
573 minHei = self.dataOut.heightList[0]
592 minHei = self.dataOut.heightList[0]
574
593
575 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
594 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
576 if self.warnings:
595 if self.warnings:
577 print('maxHei: %.2f is out of the heights range' % (maxHei))
596 print('maxHei: %.2f is out of the heights range' % (maxHei))
578 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
597 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
579 maxHei = self.dataOut.heightList[-1]
598 maxHei = self.dataOut.heightList[-1]
580
599
581
600
582 #indices relativos a los puntos de fft, puede ser de acuerdo a velocidad o frecuencia
601 #indices relativos a los puntos de fft, puede ser de acuerdo a velocidad o frecuencia
583 minIndexFFT = 0
602 minIndexFFT = 0
584 maxIndexFFT = 0
603 maxIndexFFT = 0
585 # validacion de velocidades
604 # validacion de velocidades
586 indminPoint = None
605 indminPoint = None
587 indmaxPoint = None
606 indmaxPoint = None
588 if self.dataOut.type == 'Spectra':
607 if self.dataOut.type == 'Spectra':
589 if minVel == None and maxVel == None :
608 if minVel == None and maxVel == None :
590
609
591 freqrange = self.dataOut.getFreqRange(1)
610 freqrange = self.dataOut.getFreqRange(1)
592
611
593 if minFreq == None:
612 if minFreq == None:
594 minFreq = freqrange[0]
613 minFreq = freqrange[0]
595
614
596 if maxFreq == None:
615 if maxFreq == None:
597 maxFreq = freqrange[-1]
616 maxFreq = freqrange[-1]
598
617
599 if (minFreq < freqrange[0]) or (minFreq > maxFreq):
618 if (minFreq < freqrange[0]) or (minFreq > maxFreq):
600 if self.warnings:
619 if self.warnings:
601 print('minFreq: %.2f is out of the frequency range' % (minFreq))
620 print('minFreq: %.2f is out of the frequency range' % (minFreq))
602 print('minFreq is setting to %.2f' % (freqrange[0]))
621 print('minFreq is setting to %.2f' % (freqrange[0]))
603 minFreq = freqrange[0]
622 minFreq = freqrange[0]
604
623
605 if (maxFreq > freqrange[-1]) or (maxFreq < minFreq):
624 if (maxFreq > freqrange[-1]) or (maxFreq < minFreq):
606 if self.warnings:
625 if self.warnings:
607 print('maxFreq: %.2f is out of the frequency range' % (maxFreq))
626 print('maxFreq: %.2f is out of the frequency range' % (maxFreq))
608 print('maxFreq is setting to %.2f' % (freqrange[-1]))
627 print('maxFreq is setting to %.2f' % (freqrange[-1]))
609 maxFreq = freqrange[-1]
628 maxFreq = freqrange[-1]
610
629
611 indminPoint = numpy.where(freqrange >= minFreq)
630 indminPoint = numpy.where(freqrange >= minFreq)
612 indmaxPoint = numpy.where(freqrange <= maxFreq)
631 indmaxPoint = numpy.where(freqrange <= maxFreq)
613
632
614 else:
633 else:
615
634
616 velrange = self.dataOut.getVelRange(1)
635 velrange = self.dataOut.getVelRange(1)
617
636
618 if minVel == None:
637 if minVel == None:
619 minVel = velrange[0]
638 minVel = velrange[0]
620
639
621 if maxVel == None:
640 if maxVel == None:
622 maxVel = velrange[-1]
641 maxVel = velrange[-1]
623
642
624 if (minVel < velrange[0]) or (minVel > maxVel):
643 if (minVel < velrange[0]) or (minVel > maxVel):
625 if self.warnings:
644 if self.warnings:
626 print('minVel: %.2f is out of the velocity range' % (minVel))
645 print('minVel: %.2f is out of the velocity range' % (minVel))
627 print('minVel is setting to %.2f' % (velrange[0]))
646 print('minVel is setting to %.2f' % (velrange[0]))
628 minVel = velrange[0]
647 minVel = velrange[0]
629
648
630 if (maxVel > velrange[-1]) or (maxVel < minVel):
649 if (maxVel > velrange[-1]) or (maxVel < minVel):
631 if self.warnings:
650 if self.warnings:
632 print('maxVel: %.2f is out of the velocity range' % (maxVel))
651 print('maxVel: %.2f is out of the velocity range' % (maxVel))
633 print('maxVel is setting to %.2f' % (velrange[-1]))
652 print('maxVel is setting to %.2f' % (velrange[-1]))
634 maxVel = velrange[-1]
653 maxVel = velrange[-1]
635
654
636 indminPoint = numpy.where(velrange >= minVel)
655 indminPoint = numpy.where(velrange >= minVel)
637 indmaxPoint = numpy.where(velrange <= maxVel)
656 indmaxPoint = numpy.where(velrange <= maxVel)
638
657
639
658
640 # seleccion de indices para rango
659 # seleccion de indices para rango
641 minIndex = 0
660 minIndex = 0
642 maxIndex = 0
661 maxIndex = 0
643 heights = self.dataOut.heightList
662 heights = self.dataOut.heightList
644
663
645 inda = numpy.where(heights >= minHei)
664 inda = numpy.where(heights >= minHei)
646 indb = numpy.where(heights <= maxHei)
665 indb = numpy.where(heights <= maxHei)
647
666
648 try:
667 try:
649 minIndex = inda[0][0]
668 minIndex = inda[0][0]
650 except:
669 except:
651 minIndex = 0
670 minIndex = 0
652
671
653 try:
672 try:
654 maxIndex = indb[0][-1]
673 maxIndex = indb[0][-1]
655 except:
674 except:
656 maxIndex = len(heights)
675 maxIndex = len(heights)
657
676
658 if (minIndex < 0) or (minIndex > maxIndex):
677 if (minIndex < 0) or (minIndex > maxIndex):
659 raise ValueError("some value in (%d,%d) is not valid" % (
678 raise ValueError("some value in (%d,%d) is not valid" % (
660 minIndex, maxIndex))
679 minIndex, maxIndex))
661
680
662 if (maxIndex >= self.dataOut.nHeights):
681 if (maxIndex >= self.dataOut.nHeights):
663 maxIndex = self.dataOut.nHeights - 1
682 maxIndex = self.dataOut.nHeights - 1
664 #############################################################3
683 #############################################################3
665 # seleccion de indices para velocidades
684 # seleccion de indices para velocidades
666 if self.dataOut.type == 'Spectra':
685 if self.dataOut.type == 'Spectra':
667 try:
686 try:
668 minIndexFFT = indminPoint[0][0]
687 minIndexFFT = indminPoint[0][0]
669 except:
688 except:
670 minIndexFFT = 0
689 minIndexFFT = 0
671
690
672 try:
691 try:
673 maxIndexFFT = indmaxPoint[0][-1]
692 maxIndexFFT = indmaxPoint[0][-1]
674 except:
693 except:
675 maxIndexFFT = len( self.dataOut.getFreqRange(1))
694 maxIndexFFT = len( self.dataOut.getFreqRange(1))
676
695
677 self.minIndex, self.maxIndex, self.minIndexFFT, self.maxIndexFFT = minIndex, maxIndex, minIndexFFT, maxIndexFFT
696 self.minIndex, self.maxIndex, self.minIndexFFT, self.maxIndexFFT = minIndex, maxIndex, minIndexFFT, maxIndexFFT
678 self.isConfig = True
697 self.isConfig = True
679 self.offset = 1
698 self.offset = 1
680 if offset!=None:
699 if offset!=None:
681 self.offset = 10**(offset/10)
700 self.offset = 10**(offset/10)
682 #print("config getNoiseB Done")
701 #print("config getNoiseB Done")
683
702
684 def run(self, dataOut, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
703 def run(self, dataOut, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
685 self.dataOut = dataOut
704 self.dataOut = dataOut
686
705
687 if not self.isConfig:
706 if not self.isConfig:
688 self.setup(offset, minHei, maxHei,minVel, maxVel, minFreq, maxFreq, warnings)
707 self.setup(offset, minHei, maxHei,minVel, maxVel, minFreq, maxFreq, warnings)
689
708
690 self.dataOut.noise_estimation = None
709 self.dataOut.noise_estimation = None
691 noise = None
710 noise = None
692 #print("data type: ",self.dataOut.type, self.dataOut.nIncohInt, self.dataOut.max_nIncohInt)
711 #print("data type: ",self.dataOut.type, self.dataOut.nIncohInt, self.dataOut.max_nIncohInt)
693 if self.dataOut.type == 'Voltage':
712 if self.dataOut.type == 'Voltage':
694 noise = self.dataOut.getNoise(ymin_index=self.minIndex, ymax_index=self.maxIndex)
713 noise = self.dataOut.getNoise(ymin_index=self.minIndex, ymax_index=self.maxIndex)
695 #print(minIndex, maxIndex,minIndexVel, maxIndexVel)
714 #print(minIndex, maxIndex,minIndexVel, maxIndexVel)
696 elif self.dataOut.type == 'Spectra':
715 elif self.dataOut.type == 'Spectra':
697 #print(self.dataOut.nChannels, self.minIndex, self.maxIndex,self.minIndexFFT, self.maxIndexFFT, self.dataOut.max_nIncohInt, self.dataOut.nIncohInt)
716 #print(self.dataOut.nChannels, self.minIndex, self.maxIndex,self.minIndexFFT, self.maxIndexFFT, self.dataOut.max_nIncohInt, self.dataOut.nIncohInt)
698 noise = numpy.zeros( self.dataOut.nChannels)
717 noise = numpy.zeros( self.dataOut.nChannels)
699 norm = 1
718 norm = 1
700
719
701 for channel in range( self.dataOut.nChannels):
720 for channel in range( self.dataOut.nChannels):
702 if not hasattr(self.dataOut.nIncohInt,'__len__'):
721 if not hasattr(self.dataOut.nIncohInt,'__len__'):
703 norm = 1
722 norm = 1
704 else:
723 else:
705 norm = self.dataOut.max_nIncohInt[channel]/self.dataOut.nIncohInt[channel, self.minIndex:self.maxIndex]
724 norm = self.dataOut.max_nIncohInt[channel]/self.dataOut.nIncohInt[channel, self.minIndex:self.maxIndex]
706 #print("norm nIncoh: ", norm ,self.dataOut.data_spc.shape, self.dataOut.max_nIncohInt)
725 #print("norm nIncoh: ", norm ,self.dataOut.data_spc.shape, self.dataOut.max_nIncohInt)
707 daux = self.dataOut.data_spc[channel,self.minIndexFFT:self.maxIndexFFT, self.minIndex:self.maxIndex]
726 daux = self.dataOut.data_spc[channel,self.minIndexFFT:self.maxIndexFFT, self.minIndex:self.maxIndex]
708 daux = numpy.multiply(daux, norm)
727 daux = numpy.multiply(daux, norm)
709 #print("offset: ", self.offset, 10*numpy.log10(self.offset))
728 #print("offset: ", self.offset, 10*numpy.log10(self.offset))
710 # noise[channel] = self.getNoiseByMean(daux)/self.offset
729 # noise[channel] = self.getNoiseByMean(daux)/self.offset
711 #print(daux.shape, daux)
730 #print(daux.shape, daux)
712 #noise[channel] = self.getNoiseByHS(daux, self.dataOut.max_nIncohInt)/self.offset
731 #noise[channel] = self.getNoiseByHS(daux, self.dataOut.max_nIncohInt)/self.offset
713 sortdata = numpy.sort(daux, axis=None)
732 sortdata = numpy.sort(daux, axis=None)
714
733
715 noise[channel] = _noise.hildebrand_sekhon(sortdata, self.dataOut.max_nIncohInt[channel])/self.offset
734 noise[channel] = _noise.hildebrand_sekhon(sortdata, self.dataOut.max_nIncohInt[channel])/self.offset
716
735
717
736
718 #noise = self.dataOut.getNoise(xmin_index=self.minIndexFFT, xmax_index=self.maxIndexFFT, ymin_index=self.minIndex, ymax_index=self.maxIndex)
737 #noise = self.dataOut.getNoise(xmin_index=self.minIndexFFT, xmax_index=self.maxIndexFFT, ymin_index=self.minIndex, ymax_index=self.maxIndex)
719 else:
738 else:
720 noise = self.dataOut.getNoise(xmin_index=self.minIndexFFT, xmax_index=self.maxIndexFFT, ymin_index=self.minIndex, ymax_index=self.maxIndex)
739 noise = self.dataOut.getNoise(xmin_index=self.minIndexFFT, xmax_index=self.maxIndexFFT, ymin_index=self.minIndex, ymax_index=self.maxIndex)
721 self.dataOut.noise_estimation = noise.copy() # dataOut.noise
740 self.dataOut.noise_estimation = noise.copy() # dataOut.noise
722 #print("2: ",10*numpy.log10(self.dataOut.noise_estimation/64))
741 #print("2: ",10*numpy.log10(self.dataOut.noise_estimation/64))
723 #print("2: ",self.dataOut.noise_estimation)
742 #print("2: ",self.dataOut.noise_estimation)
724 #print(self.dataOut.flagNoData)
743 #print(self.dataOut.flagNoData)
725 #print("getNoise Done", noise, self.dataOut.nProfiles ,self.dataOut.ippFactor)
744 #print("getNoise Done", noise, self.dataOut.nProfiles ,self.dataOut.ippFactor)
726 return self.dataOut
745 return self.dataOut
727
746
728 def getNoiseByMean(self,data):
747 def getNoiseByMean(self,data):
729 #data debe estar ordenado
748 #data debe estar ordenado
730 data = numpy.mean(data,axis=1)
749 data = numpy.mean(data,axis=1)
731 sortdata = numpy.sort(data, axis=None)
750 sortdata = numpy.sort(data, axis=None)
732 #sortID=data.argsort()
751 #sortID=data.argsort()
733 #print(data.shape)
752 #print(data.shape)
734
753
735 pnoise = None
754 pnoise = None
736 j = 0
755 j = 0
737
756
738 mean = numpy.mean(sortdata)
757 mean = numpy.mean(sortdata)
739 min = numpy.min(sortdata)
758 min = numpy.min(sortdata)
740 delta = mean - min
759 delta = mean - min
741 indexes = numpy.where(sortdata > (mean+delta))[0] #only array of indexes
760 indexes = numpy.where(sortdata > (mean+delta))[0] #only array of indexes
742 #print(len(indexes))
761 #print(len(indexes))
743 if len(indexes)==0:
762 if len(indexes)==0:
744 pnoise = numpy.mean(sortdata)
763 pnoise = numpy.mean(sortdata)
745 else:
764 else:
746 j = indexes[0]
765 j = indexes[0]
747 pnoise = numpy.mean(sortdata[0:j])
766 pnoise = numpy.mean(sortdata[0:j])
748
767
749 # from matplotlib import pyplot as plt
768 # from matplotlib import pyplot as plt
750 # plt.plot(sortdata)
769 # plt.plot(sortdata)
751 # plt.vlines(j,(pnoise-delta),(pnoise+delta), color='r')
770 # plt.vlines(j,(pnoise-delta),(pnoise+delta), color='r')
752 # plt.show()
771 # plt.show()
753 #print("noise: ", 10*numpy.log10(pnoise))
772 #print("noise: ", 10*numpy.log10(pnoise))
754 return pnoise
773 return pnoise
755
774
756 def getNoiseByHS(self,data, navg):
775 def getNoiseByHS(self,data, navg):
757 #data debe estar ordenado
776 #data debe estar ordenado
758 #data = numpy.mean(data,axis=1)
777 #data = numpy.mean(data,axis=1)
759 sortdata = numpy.sort(data, axis=None)
778 sortdata = numpy.sort(data, axis=None)
760
779
761 lenOfData = len(sortdata)
780 lenOfData = len(sortdata)
762 nums_min = lenOfData*0.2
781 nums_min = lenOfData*0.2
763
782
764 if nums_min <= 5:
783 if nums_min <= 5:
765
784
766 nums_min = 5
785 nums_min = 5
767
786
768 sump = 0.
787 sump = 0.
769 sumq = 0.
788 sumq = 0.
770
789
771 j = 0
790 j = 0
772 cont = 1
791 cont = 1
773
792
774 while((cont == 1)and(j < lenOfData)):
793 while((cont == 1)and(j < lenOfData)):
775
794
776 sump += sortdata[j]
795 sump += sortdata[j]
777 sumq += sortdata[j]**2
796 sumq += sortdata[j]**2
778 #sumq -= sump**2
797 #sumq -= sump**2
779 if j > nums_min:
798 if j > nums_min:
780 rtest = float(j)/(j-1) + 1.0/navg
799 rtest = float(j)/(j-1) + 1.0/navg
781 #if ((sumq*j) > (sump**2)):
800 #if ((sumq*j) > (sump**2)):
782 if ((sumq*j) > (rtest*sump**2)):
801 if ((sumq*j) > (rtest*sump**2)):
783 j = j - 1
802 j = j - 1
784 sump = sump - sortdata[j]
803 sump = sump - sortdata[j]
785 sumq = sumq - sortdata[j]**2
804 sumq = sumq - sortdata[j]**2
786 cont = 0
805 cont = 0
787
806
788 j += 1
807 j += 1
789
808
790 lnoise = sump / j
809 lnoise = sump / j
791
810
792 return lnoise
811 return lnoise
793
812
794
813
795
814
796 def fit_func( x, a0, a1, a2): #, a3, a4, a5):
815 def fit_func( x, a0, a1, a2): #, a3, a4, a5):
797 z = (x - a1) / a2
816 z = (x - a1) / a2
798 y = a0 * numpy.exp(-z**2 / a2) #+ a3 + a4 * x + a5 * x**2
817 y = a0 * numpy.exp(-z**2 / a2) #+ a3 + a4 * x + a5 * x**2
799 return y
818 return y
800
819
801
820
802 # class CleanRayleigh(Operation):
821 # class CleanRayleigh(Operation):
803 #
822 #
804 # def __init__(self):
823 # def __init__(self):
805 #
824 #
806 # Operation.__init__(self)
825 # Operation.__init__(self)
807 # self.i=0
826 # self.i=0
808 # self.isConfig = False
827 # self.isConfig = False
809 # self.__dataReady = False
828 # self.__dataReady = False
810 # self.__profIndex = 0
829 # self.__profIndex = 0
811 # self.byTime = False
830 # self.byTime = False
812 # self.byProfiles = False
831 # self.byProfiles = False
813 #
832 #
814 # self.bloques = None
833 # self.bloques = None
815 # self.bloque0 = None
834 # self.bloque0 = None
816 #
835 #
817 # self.index = 0
836 # self.index = 0
818 #
837 #
819 # self.buffer = 0
838 # self.buffer = 0
820 # self.buffer2 = 0
839 # self.buffer2 = 0
821 # self.buffer3 = 0
840 # self.buffer3 = 0
822 #
841 #
823 #
842 #
824 # def setup(self,dataOut,min_hei,max_hei,n, timeInterval,factor_stdv):
843 # def setup(self,dataOut,min_hei,max_hei,n, timeInterval,factor_stdv):
825 #
844 #
826 # self.nChannels = dataOut.nChannels
845 # self.nChannels = dataOut.nChannels
827 # self.nProf = dataOut.nProfiles
846 # self.nProf = dataOut.nProfiles
828 # self.nPairs = dataOut.data_cspc.shape[0]
847 # self.nPairs = dataOut.data_cspc.shape[0]
829 # self.pairsArray = numpy.array(dataOut.pairsList)
848 # self.pairsArray = numpy.array(dataOut.pairsList)
830 # self.spectra = dataOut.data_spc
849 # self.spectra = dataOut.data_spc
831 # self.cspectra = dataOut.data_cspc
850 # self.cspectra = dataOut.data_cspc
832 # self.heights = dataOut.heightList #alturas totales
851 # self.heights = dataOut.heightList #alturas totales
833 # self.nHeights = len(self.heights)
852 # self.nHeights = len(self.heights)
834 # self.min_hei = min_hei
853 # self.min_hei = min_hei
835 # self.max_hei = max_hei
854 # self.max_hei = max_hei
836 # if (self.min_hei == None):
855 # if (self.min_hei == None):
837 # self.min_hei = 0
856 # self.min_hei = 0
838 # if (self.max_hei == None):
857 # if (self.max_hei == None):
839 # self.max_hei = dataOut.heightList[-1]
858 # self.max_hei = dataOut.heightList[-1]
840 # self.hval = ((self.max_hei>=self.heights) & (self.heights >= self.min_hei)).nonzero()
859 # self.hval = ((self.max_hei>=self.heights) & (self.heights >= self.min_hei)).nonzero()
841 # self.heightsClean = self.heights[self.hval] #alturas filtradas
860 # self.heightsClean = self.heights[self.hval] #alturas filtradas
842 # self.hval = self.hval[0] # forma (N,), an solo N elementos -> Indices de alturas
861 # self.hval = self.hval[0] # forma (N,), an solo N elementos -> Indices de alturas
843 # self.nHeightsClean = len(self.heightsClean)
862 # self.nHeightsClean = len(self.heightsClean)
844 # self.channels = dataOut.channelList
863 # self.channels = dataOut.channelList
845 # self.nChan = len(self.channels)
864 # self.nChan = len(self.channels)
846 # self.nIncohInt = dataOut.nIncohInt
865 # self.nIncohInt = dataOut.nIncohInt
847 # self.__initime = dataOut.utctime
866 # self.__initime = dataOut.utctime
848 # self.maxAltInd = self.hval[-1]+1
867 # self.maxAltInd = self.hval[-1]+1
849 # self.minAltInd = self.hval[0]
868 # self.minAltInd = self.hval[0]
850 #
869 #
851 # self.crosspairs = dataOut.pairsList
870 # self.crosspairs = dataOut.pairsList
852 # self.nPairs = len(self.crosspairs)
871 # self.nPairs = len(self.crosspairs)
853 # self.normFactor = dataOut.normFactor
872 # self.normFactor = dataOut.normFactor
854 # self.nFFTPoints = dataOut.nFFTPoints
873 # self.nFFTPoints = dataOut.nFFTPoints
855 # self.ippSeconds = dataOut.ippSeconds
874 # self.ippSeconds = dataOut.ippSeconds
856 # self.currentTime = self.__initime
875 # self.currentTime = self.__initime
857 # self.pairsArray = numpy.array(dataOut.pairsList)
876 # self.pairsArray = numpy.array(dataOut.pairsList)
858 # self.factor_stdv = factor_stdv
877 # self.factor_stdv = factor_stdv
859 #
878 #
860 # if n != None :
879 # if n != None :
861 # self.byProfiles = True
880 # self.byProfiles = True
862 # self.nIntProfiles = n
881 # self.nIntProfiles = n
863 # else:
882 # else:
864 # self.__integrationtime = timeInterval
883 # self.__integrationtime = timeInterval
865 #
884 #
866 # self.__dataReady = False
885 # self.__dataReady = False
867 # self.isConfig = True
886 # self.isConfig = True
868 #
887 #
869 #
888 #
870 #
889 #
871 # def run(self, dataOut,min_hei=None,max_hei=None, n=None, timeInterval=10,factor_stdv=2.5):
890 # def run(self, dataOut,min_hei=None,max_hei=None, n=None, timeInterval=10,factor_stdv=2.5):
872 # #print("runing cleanRayleigh")
891 # #print("runing cleanRayleigh")
873 # if not self.isConfig :
892 # if not self.isConfig :
874 #
893 #
875 # self.setup(dataOut, min_hei,max_hei,n,timeInterval,factor_stdv)
894 # self.setup(dataOut, min_hei,max_hei,n,timeInterval,factor_stdv)
876 #
895 #
877 # tini=dataOut.utctime
896 # tini=dataOut.utctime
878 #
897 #
879 # if self.byProfiles:
898 # if self.byProfiles:
880 # if self.__profIndex == self.nIntProfiles:
899 # if self.__profIndex == self.nIntProfiles:
881 # self.__dataReady = True
900 # self.__dataReady = True
882 # else:
901 # else:
883 # if (tini - self.__initime) >= self.__integrationtime:
902 # if (tini - self.__initime) >= self.__integrationtime:
884 #
903 #
885 # self.__dataReady = True
904 # self.__dataReady = True
886 # self.__initime = tini
905 # self.__initime = tini
887 #
906 #
888 # #if (tini.tm_min % 2) == 0 and (tini.tm_sec < 5 and self.fint==0):
907 # #if (tini.tm_min % 2) == 0 and (tini.tm_sec < 5 and self.fint==0):
889 #
908 #
890 # if self.__dataReady:
909 # if self.__dataReady:
891 #
910 #
892 # self.__profIndex = 0
911 # self.__profIndex = 0
893 # jspc = self.buffer
912 # jspc = self.buffer
894 # jcspc = self.buffer2
913 # jcspc = self.buffer2
895 # #jnoise = self.buffer3
914 # #jnoise = self.buffer3
896 # self.buffer = dataOut.data_spc
915 # self.buffer = dataOut.data_spc
897 # self.buffer2 = dataOut.data_cspc
916 # self.buffer2 = dataOut.data_cspc
898 # #self.buffer3 = dataOut.noise
917 # #self.buffer3 = dataOut.noise
899 # self.currentTime = dataOut.utctime
918 # self.currentTime = dataOut.utctime
900 # if numpy.any(jspc) :
919 # if numpy.any(jspc) :
901 # #print( jspc.shape, jcspc.shape)
920 # #print( jspc.shape, jcspc.shape)
902 # jspc = numpy.reshape(jspc,(int(len(jspc)/self.nChannels),self.nChannels,self.nFFTPoints,self.nHeights))
921 # jspc = numpy.reshape(jspc,(int(len(jspc)/self.nChannels),self.nChannels,self.nFFTPoints,self.nHeights))
903 # try:
922 # try:
904 # jcspc= numpy.reshape(jcspc,(int(len(jcspc)/self.nPairs),self.nPairs,self.nFFTPoints,self.nHeights))
923 # jcspc= numpy.reshape(jcspc,(int(len(jcspc)/self.nPairs),self.nPairs,self.nFFTPoints,self.nHeights))
905 # except:
924 # except:
906 # print("no cspc")
925 # print("no cspc")
907 # self.__dataReady = False
926 # self.__dataReady = False
908 # #print( jspc.shape, jcspc.shape)
927 # #print( jspc.shape, jcspc.shape)
909 # dataOut.flagNoData = False
928 # dataOut.flagNoData = False
910 # else:
929 # else:
911 # dataOut.flagNoData = True
930 # dataOut.flagNoData = True
912 # self.__dataReady = False
931 # self.__dataReady = False
913 # return dataOut
932 # return dataOut
914 # else:
933 # else:
915 # #print( len(self.buffer))
934 # #print( len(self.buffer))
916 # if numpy.any(self.buffer):
935 # if numpy.any(self.buffer):
917 # self.buffer = numpy.concatenate((self.buffer,dataOut.data_spc), axis=0)
936 # self.buffer = numpy.concatenate((self.buffer,dataOut.data_spc), axis=0)
918 # try:
937 # try:
919 # self.buffer2 = numpy.concatenate((self.buffer2,dataOut.data_cspc), axis=0)
938 # self.buffer2 = numpy.concatenate((self.buffer2,dataOut.data_cspc), axis=0)
920 # self.buffer3 += dataOut.data_dc
939 # self.buffer3 += dataOut.data_dc
921 # except:
940 # except:
922 # pass
941 # pass
923 # else:
942 # else:
924 # self.buffer = dataOut.data_spc
943 # self.buffer = dataOut.data_spc
925 # self.buffer2 = dataOut.data_cspc
944 # self.buffer2 = dataOut.data_cspc
926 # self.buffer3 = dataOut.data_dc
945 # self.buffer3 = dataOut.data_dc
927 # #print self.index, self.fint
946 # #print self.index, self.fint
928 # #print self.buffer2.shape
947 # #print self.buffer2.shape
929 # dataOut.flagNoData = True ## NOTE: ?? revisar LUEGO
948 # dataOut.flagNoData = True ## NOTE: ?? revisar LUEGO
930 # self.__profIndex += 1
949 # self.__profIndex += 1
931 # return dataOut ## NOTE: REV
950 # return dataOut ## NOTE: REV
932 #
951 #
933 #
952 #
934 # #index = tini.tm_hour*12+tini.tm_min/5
953 # #index = tini.tm_hour*12+tini.tm_min/5
935 # '''
954 # '''
936 # #REVISAR
955 # #REVISAR
937 # '''
956 # '''
938 # # jspc = jspc/self.nFFTPoints/self.normFactor
957 # # jspc = jspc/self.nFFTPoints/self.normFactor
939 # # jcspc = jcspc/self.nFFTPoints/self.normFactor
958 # # jcspc = jcspc/self.nFFTPoints/self.normFactor
940 #
959 #
941 #
960 #
942 #
961 #
943 # tmp_spectra,tmp_cspectra = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
962 # tmp_spectra,tmp_cspectra = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
944 # dataOut.data_spc = tmp_spectra
963 # dataOut.data_spc = tmp_spectra
945 # dataOut.data_cspc = tmp_cspectra
964 # dataOut.data_cspc = tmp_cspectra
946 #
965 #
947 # #dataOut.data_spc,dataOut.data_cspc = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
966 # #dataOut.data_spc,dataOut.data_cspc = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
948 #
967 #
949 # dataOut.data_dc = self.buffer3
968 # dataOut.data_dc = self.buffer3
950 # dataOut.nIncohInt *= self.nIntProfiles
969 # dataOut.nIncohInt *= self.nIntProfiles
951 # dataOut.max_nIncohInt = self.nIntProfiles
970 # dataOut.max_nIncohInt = self.nIntProfiles
952 # dataOut.utctime = self.currentTime #tiempo promediado
971 # dataOut.utctime = self.currentTime #tiempo promediado
953 # #print("Time: ",time.localtime(dataOut.utctime))
972 # #print("Time: ",time.localtime(dataOut.utctime))
954 # # dataOut.data_spc = sat_spectra
973 # # dataOut.data_spc = sat_spectra
955 # # dataOut.data_cspc = sat_cspectra
974 # # dataOut.data_cspc = sat_cspectra
956 # self.buffer = 0
975 # self.buffer = 0
957 # self.buffer2 = 0
976 # self.buffer2 = 0
958 # self.buffer3 = 0
977 # self.buffer3 = 0
959 #
978 #
960 # return dataOut
979 # return dataOut
961 #
980 #
962 # def cleanRayleigh(self,dataOut,spectra,cspectra,factor_stdv):
981 # def cleanRayleigh(self,dataOut,spectra,cspectra,factor_stdv):
963 # print("OP cleanRayleigh")
982 # print("OP cleanRayleigh")
964 # #import matplotlib.pyplot as plt
983 # #import matplotlib.pyplot as plt
965 # #for k in range(149):
984 # #for k in range(149):
966 # #channelsProcssd = []
985 # #channelsProcssd = []
967 # #channelA_ok = False
986 # #channelA_ok = False
968 # #rfunc = cspectra.copy() #self.bloques
987 # #rfunc = cspectra.copy() #self.bloques
969 # rfunc = spectra.copy()
988 # rfunc = spectra.copy()
970 # #rfunc = cspectra
989 # #rfunc = cspectra
971 # #val_spc = spectra*0.0 #self.bloque0*0.0
990 # #val_spc = spectra*0.0 #self.bloque0*0.0
972 # #val_cspc = cspectra*0.0 #self.bloques*0.0
991 # #val_cspc = cspectra*0.0 #self.bloques*0.0
973 # #in_sat_spectra = spectra.copy() #self.bloque0
992 # #in_sat_spectra = spectra.copy() #self.bloque0
974 # #in_sat_cspectra = cspectra.copy() #self.bloques
993 # #in_sat_cspectra = cspectra.copy() #self.bloques
975 #
994 #
976 #
995 #
977 # ###ONLY FOR TEST:
996 # ###ONLY FOR TEST:
978 # raxs = math.ceil(math.sqrt(self.nPairs))
997 # raxs = math.ceil(math.sqrt(self.nPairs))
979 # if raxs == 0:
998 # if raxs == 0:
980 # raxs = 1
999 # raxs = 1
981 # caxs = math.ceil(self.nPairs/raxs)
1000 # caxs = math.ceil(self.nPairs/raxs)
982 # if self.nPairs <4:
1001 # if self.nPairs <4:
983 # raxs = 2
1002 # raxs = 2
984 # caxs = 2
1003 # caxs = 2
985 # #print(raxs, caxs)
1004 # #print(raxs, caxs)
986 # fft_rev = 14 #nFFT to plot
1005 # fft_rev = 14 #nFFT to plot
987 # hei_rev = ((self.heights >= 550) & (self.heights <= 551)).nonzero() #hei to plot
1006 # hei_rev = ((self.heights >= 550) & (self.heights <= 551)).nonzero() #hei to plot
988 # hei_rev = hei_rev[0]
1007 # hei_rev = hei_rev[0]
989 # #print(hei_rev)
1008 # #print(hei_rev)
990 #
1009 #
991 # #print numpy.absolute(rfunc[:,0,0,14])
1010 # #print numpy.absolute(rfunc[:,0,0,14])
992 #
1011 #
993 # gauss_fit, covariance = None, None
1012 # gauss_fit, covariance = None, None
994 # for ih in range(self.minAltInd,self.maxAltInd):
1013 # for ih in range(self.minAltInd,self.maxAltInd):
995 # for ifreq in range(self.nFFTPoints):
1014 # for ifreq in range(self.nFFTPoints):
996 # '''
1015 # '''
997 # ###ONLY FOR TEST:
1016 # ###ONLY FOR TEST:
998 # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1017 # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
999 # fig, axs = plt.subplots(raxs, caxs)
1018 # fig, axs = plt.subplots(raxs, caxs)
1000 # fig2, axs2 = plt.subplots(raxs, caxs)
1019 # fig2, axs2 = plt.subplots(raxs, caxs)
1001 # col_ax = 0
1020 # col_ax = 0
1002 # row_ax = 0
1021 # row_ax = 0
1003 # '''
1022 # '''
1004 # #print(self.nPairs)
1023 # #print(self.nPairs)
1005 # for ii in range(self.nChan): #PARES DE CANALES SELF y CROSS
1024 # for ii in range(self.nChan): #PARES DE CANALES SELF y CROSS
1006 # # if self.crosspairs[ii][1]-self.crosspairs[ii][0] > 1: # APLICAR SOLO EN PARES CONTIGUOS
1025 # # if self.crosspairs[ii][1]-self.crosspairs[ii][0] > 1: # APLICAR SOLO EN PARES CONTIGUOS
1007 # # continue
1026 # # continue
1008 # # if not self.crosspairs[ii][0] in channelsProcssd:
1027 # # if not self.crosspairs[ii][0] in channelsProcssd:
1009 # # channelA_ok = True
1028 # # channelA_ok = True
1010 # #print("pair: ",self.crosspairs[ii])
1029 # #print("pair: ",self.crosspairs[ii])
1011 # '''
1030 # '''
1012 # ###ONLY FOR TEST:
1031 # ###ONLY FOR TEST:
1013 # if (col_ax%caxs==0 and col_ax!=0 and self.nPairs !=1):
1032 # if (col_ax%caxs==0 and col_ax!=0 and self.nPairs !=1):
1014 # col_ax = 0
1033 # col_ax = 0
1015 # row_ax += 1
1034 # row_ax += 1
1016 # '''
1035 # '''
1017 # func2clean = 10*numpy.log10(numpy.absolute(rfunc[:,ii,ifreq,ih])) #Potencia?
1036 # func2clean = 10*numpy.log10(numpy.absolute(rfunc[:,ii,ifreq,ih])) #Potencia?
1018 # #print(func2clean.shape)
1037 # #print(func2clean.shape)
1019 # val = (numpy.isfinite(func2clean)==True).nonzero()
1038 # val = (numpy.isfinite(func2clean)==True).nonzero()
1020 #
1039 #
1021 # if len(val)>0: #limitador
1040 # if len(val)>0: #limitador
1022 # min_val = numpy.around(numpy.amin(func2clean)-2) #> (-40)
1041 # min_val = numpy.around(numpy.amin(func2clean)-2) #> (-40)
1023 # if min_val <= -40 :
1042 # if min_val <= -40 :
1024 # min_val = -40
1043 # min_val = -40
1025 # max_val = numpy.around(numpy.amax(func2clean)+2) #< 200
1044 # max_val = numpy.around(numpy.amax(func2clean)+2) #< 200
1026 # if max_val >= 200 :
1045 # if max_val >= 200 :
1027 # max_val = 200
1046 # max_val = 200
1028 # #print min_val, max_val
1047 # #print min_val, max_val
1029 # step = 1
1048 # step = 1
1030 # #print("Getting bins and the histogram")
1049 # #print("Getting bins and the histogram")
1031 # x_dist = min_val + numpy.arange(1 + ((max_val-(min_val))/step))*step
1050 # x_dist = min_val + numpy.arange(1 + ((max_val-(min_val))/step))*step
1032 # y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
1051 # y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
1033 # #print(len(y_dist),len(binstep[:-1]))
1052 # #print(len(y_dist),len(binstep[:-1]))
1034 # #print(row_ax,col_ax, " ..")
1053 # #print(row_ax,col_ax, " ..")
1035 # #print(self.pairsArray[ii][0],self.pairsArray[ii][1])
1054 # #print(self.pairsArray[ii][0],self.pairsArray[ii][1])
1036 # mean = numpy.sum(x_dist * y_dist) / numpy.sum(y_dist)
1055 # mean = numpy.sum(x_dist * y_dist) / numpy.sum(y_dist)
1037 # sigma = numpy.sqrt(numpy.sum(y_dist * (x_dist - mean)**2) / numpy.sum(y_dist))
1056 # sigma = numpy.sqrt(numpy.sum(y_dist * (x_dist - mean)**2) / numpy.sum(y_dist))
1038 # parg = [numpy.amax(y_dist),mean,sigma]
1057 # parg = [numpy.amax(y_dist),mean,sigma]
1039 #
1058 #
1040 # newY = None
1059 # newY = None
1041 #
1060 #
1042 # try :
1061 # try :
1043 # gauss_fit, covariance = curve_fit(fit_func, x_dist, y_dist,p0=parg)
1062 # gauss_fit, covariance = curve_fit(fit_func, x_dist, y_dist,p0=parg)
1044 # mode = gauss_fit[1]
1063 # mode = gauss_fit[1]
1045 # stdv = gauss_fit[2]
1064 # stdv = gauss_fit[2]
1046 # #print(" FIT OK",gauss_fit)
1065 # #print(" FIT OK",gauss_fit)
1047 # '''
1066 # '''
1048 # ###ONLY FOR TEST:
1067 # ###ONLY FOR TEST:
1049 # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1068 # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1050 # newY = fit_func(x_dist,gauss_fit[0],gauss_fit[1],gauss_fit[2])
1069 # newY = fit_func(x_dist,gauss_fit[0],gauss_fit[1],gauss_fit[2])
1051 # axs[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
1070 # axs[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
1052 # axs[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
1071 # axs[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
1053 # axs[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
1072 # axs[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
1054 # '''
1073 # '''
1055 # except:
1074 # except:
1056 # mode = mean
1075 # mode = mean
1057 # stdv = sigma
1076 # stdv = sigma
1058 # #print("FIT FAIL")
1077 # #print("FIT FAIL")
1059 # #continue
1078 # #continue
1060 #
1079 #
1061 #
1080 #
1062 # #print(mode,stdv)
1081 # #print(mode,stdv)
1063 # #Removing echoes greater than mode + std_factor*stdv
1082 # #Removing echoes greater than mode + std_factor*stdv
1064 # noval = (abs(func2clean - mode)>=(factor_stdv*stdv)).nonzero()
1083 # noval = (abs(func2clean - mode)>=(factor_stdv*stdv)).nonzero()
1065 # #noval tiene los indices que se van a remover
1084 # #noval tiene los indices que se van a remover
1066 # #print("Chan ",ii," novals: ",len(noval[0]))
1085 # #print("Chan ",ii," novals: ",len(noval[0]))
1067 # if len(noval[0]) > 0: #forma de array (N,) es igual a longitud (N)
1086 # if len(noval[0]) > 0: #forma de array (N,) es igual a longitud (N)
1068 # novall = ((func2clean - mode) >= (factor_stdv*stdv)).nonzero()
1087 # novall = ((func2clean - mode) >= (factor_stdv*stdv)).nonzero()
1069 # #print(novall)
1088 # #print(novall)
1070 # #print(" ",self.pairsArray[ii])
1089 # #print(" ",self.pairsArray[ii])
1071 # #cross_pairs = self.pairsArray[ii]
1090 # #cross_pairs = self.pairsArray[ii]
1072 # #Getting coherent echoes which are removed.
1091 # #Getting coherent echoes which are removed.
1073 # # if len(novall[0]) > 0:
1092 # # if len(novall[0]) > 0:
1074 # #
1093 # #
1075 # # val_spc[novall[0],cross_pairs[0],ifreq,ih] = 1
1094 # # val_spc[novall[0],cross_pairs[0],ifreq,ih] = 1
1076 # # val_spc[novall[0],cross_pairs[1],ifreq,ih] = 1
1095 # # val_spc[novall[0],cross_pairs[1],ifreq,ih] = 1
1077 # # val_cspc[novall[0],ii,ifreq,ih] = 1
1096 # # val_cspc[novall[0],ii,ifreq,ih] = 1
1078 # #print("OUT NOVALL 1")
1097 # #print("OUT NOVALL 1")
1079 # try:
1098 # try:
1080 # pair = (self.channels[ii],self.channels[ii + 1])
1099 # pair = (self.channels[ii],self.channels[ii + 1])
1081 # except:
1100 # except:
1082 # pair = (99,99)
1101 # pair = (99,99)
1083 # #print("par ", pair)
1102 # #print("par ", pair)
1084 # if ( pair in self.crosspairs):
1103 # if ( pair in self.crosspairs):
1085 # q = self.crosspairs.index(pair)
1104 # q = self.crosspairs.index(pair)
1086 # #print("está aqui: ", q, (ii,ii + 1))
1105 # #print("está aqui: ", q, (ii,ii + 1))
1087 # new_a = numpy.delete(cspectra[:,q,ifreq,ih], noval[0])
1106 # new_a = numpy.delete(cspectra[:,q,ifreq,ih], noval[0])
1088 # cspectra[noval,q,ifreq,ih] = numpy.mean(new_a) #mean CrossSpectra
1107 # cspectra[noval,q,ifreq,ih] = numpy.mean(new_a) #mean CrossSpectra
1089 #
1108 #
1090 # #if channelA_ok:
1109 # #if channelA_ok:
1091 # #chA = self.channels.index(cross_pairs[0])
1110 # #chA = self.channels.index(cross_pairs[0])
1092 # new_b = numpy.delete(spectra[:,ii,ifreq,ih], noval[0])
1111 # new_b = numpy.delete(spectra[:,ii,ifreq,ih], noval[0])
1093 # spectra[noval,ii,ifreq,ih] = numpy.mean(new_b) #mean Spectra Pair A
1112 # spectra[noval,ii,ifreq,ih] = numpy.mean(new_b) #mean Spectra Pair A
1094 # #channelA_ok = False
1113 # #channelA_ok = False
1095 #
1114 #
1096 # # chB = self.channels.index(cross_pairs[1])
1115 # # chB = self.channels.index(cross_pairs[1])
1097 # # new_c = numpy.delete(spectra[:,chB,ifreq,ih], noval[0])
1116 # # new_c = numpy.delete(spectra[:,chB,ifreq,ih], noval[0])
1098 # # spectra[noval,chB,ifreq,ih] = numpy.mean(new_c) #mean Spectra Pair B
1117 # # spectra[noval,chB,ifreq,ih] = numpy.mean(new_c) #mean Spectra Pair B
1099 # #
1118 # #
1100 # # channelsProcssd.append(self.crosspairs[ii][0]) # save channel A
1119 # # channelsProcssd.append(self.crosspairs[ii][0]) # save channel A
1101 # # channelsProcssd.append(self.crosspairs[ii][1]) # save channel B
1120 # # channelsProcssd.append(self.crosspairs[ii][1]) # save channel B
1102 # '''
1121 # '''
1103 # ###ONLY FOR TEST:
1122 # ###ONLY FOR TEST:
1104 # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1123 # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1105 # func2clean = 10*numpy.log10(numpy.absolute(spectra[:,ii,ifreq,ih]))
1124 # func2clean = 10*numpy.log10(numpy.absolute(spectra[:,ii,ifreq,ih]))
1106 # y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
1125 # y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
1107 # axs2[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
1126 # axs2[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
1108 # axs2[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
1127 # axs2[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
1109 # axs2[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
1128 # axs2[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
1110 # '''
1129 # '''
1111 # '''
1130 # '''
1112 # ###ONLY FOR TEST:
1131 # ###ONLY FOR TEST:
1113 # col_ax += 1 #contador de ploteo columnas
1132 # col_ax += 1 #contador de ploteo columnas
1114 # ##print(col_ax)
1133 # ##print(col_ax)
1115 # ###ONLY FOR TEST:
1134 # ###ONLY FOR TEST:
1116 # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1135 # if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
1117 # title = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km"
1136 # title = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km"
1118 # title2 = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km CLEANED"
1137 # title2 = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km CLEANED"
1119 # fig.suptitle(title)
1138 # fig.suptitle(title)
1120 # fig2.suptitle(title2)
1139 # fig2.suptitle(title2)
1121 # plt.show()
1140 # plt.show()
1122 # '''
1141 # '''
1123 # ##################################################################################################
1142 # ##################################################################################################
1124 #
1143 #
1125 # #print("Getting average of the spectra and cross-spectra from incoherent echoes.")
1144 # #print("Getting average of the spectra and cross-spectra from incoherent echoes.")
1126 # out_spectra = numpy.zeros([self.nChan,self.nFFTPoints,self.nHeights], dtype=float) #+numpy.nan
1145 # out_spectra = numpy.zeros([self.nChan,self.nFFTPoints,self.nHeights], dtype=float) #+numpy.nan
1127 # out_cspectra = numpy.zeros([self.nPairs,self.nFFTPoints,self.nHeights], dtype=complex) #+numpy.nan
1146 # out_cspectra = numpy.zeros([self.nPairs,self.nFFTPoints,self.nHeights], dtype=complex) #+numpy.nan
1128 # for ih in range(self.nHeights):
1147 # for ih in range(self.nHeights):
1129 # for ifreq in range(self.nFFTPoints):
1148 # for ifreq in range(self.nFFTPoints):
1130 # for ich in range(self.nChan):
1149 # for ich in range(self.nChan):
1131 # tmp = spectra[:,ich,ifreq,ih]
1150 # tmp = spectra[:,ich,ifreq,ih]
1132 # valid = (numpy.isfinite(tmp[:])==True).nonzero()
1151 # valid = (numpy.isfinite(tmp[:])==True).nonzero()
1133 #
1152 #
1134 # if len(valid[0]) >0 :
1153 # if len(valid[0]) >0 :
1135 # out_spectra[ich,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
1154 # out_spectra[ich,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
1136 #
1155 #
1137 # for icr in range(self.nPairs):
1156 # for icr in range(self.nPairs):
1138 # tmp = numpy.squeeze(cspectra[:,icr,ifreq,ih])
1157 # tmp = numpy.squeeze(cspectra[:,icr,ifreq,ih])
1139 # valid = (numpy.isfinite(tmp)==True).nonzero()
1158 # valid = (numpy.isfinite(tmp)==True).nonzero()
1140 # if len(valid[0]) > 0:
1159 # if len(valid[0]) > 0:
1141 # out_cspectra[icr,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
1160 # out_cspectra[icr,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
1142 #
1161 #
1143 # return out_spectra, out_cspectra
1162 # return out_spectra, out_cspectra
1144 #
1163 #
1145 # def REM_ISOLATED_POINTS(self,array,rth):
1164 # def REM_ISOLATED_POINTS(self,array,rth):
1146 # # import matplotlib.pyplot as plt
1165 # # import matplotlib.pyplot as plt
1147 # if rth == None :
1166 # if rth == None :
1148 # rth = 4
1167 # rth = 4
1149 # #print("REM ISO")
1168 # #print("REM ISO")
1150 # num_prof = len(array[0,:,0])
1169 # num_prof = len(array[0,:,0])
1151 # num_hei = len(array[0,0,:])
1170 # num_hei = len(array[0,0,:])
1152 # n2d = len(array[:,0,0])
1171 # n2d = len(array[:,0,0])
1153 #
1172 #
1154 # for ii in range(n2d) :
1173 # for ii in range(n2d) :
1155 # #print ii,n2d
1174 # #print ii,n2d
1156 # tmp = array[ii,:,:]
1175 # tmp = array[ii,:,:]
1157 # #print tmp.shape, array[ii,101,:],array[ii,102,:]
1176 # #print tmp.shape, array[ii,101,:],array[ii,102,:]
1158 #
1177 #
1159 # # fig = plt.figure(figsize=(6,5))
1178 # # fig = plt.figure(figsize=(6,5))
1160 # # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
1179 # # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
1161 # # ax = fig.add_axes([left, bottom, width, height])
1180 # # ax = fig.add_axes([left, bottom, width, height])
1162 # # x = range(num_prof)
1181 # # x = range(num_prof)
1163 # # y = range(num_hei)
1182 # # y = range(num_hei)
1164 # # cp = ax.contour(y,x,tmp)
1183 # # cp = ax.contour(y,x,tmp)
1165 # # ax.clabel(cp, inline=True,fontsize=10)
1184 # # ax.clabel(cp, inline=True,fontsize=10)
1166 # # plt.show()
1185 # # plt.show()
1167 #
1186 #
1168 # #indxs = WHERE(FINITE(tmp) AND tmp GT 0,cindxs)
1187 # #indxs = WHERE(FINITE(tmp) AND tmp GT 0,cindxs)
1169 # tmp = numpy.reshape(tmp,num_prof*num_hei)
1188 # tmp = numpy.reshape(tmp,num_prof*num_hei)
1170 # indxs1 = (numpy.isfinite(tmp)==True).nonzero()
1189 # indxs1 = (numpy.isfinite(tmp)==True).nonzero()
1171 # indxs2 = (tmp > 0).nonzero()
1190 # indxs2 = (tmp > 0).nonzero()
1172 #
1191 #
1173 # indxs1 = (indxs1[0])
1192 # indxs1 = (indxs1[0])
1174 # indxs2 = indxs2[0]
1193 # indxs2 = indxs2[0]
1175 # #indxs1 = numpy.array(indxs1[0])
1194 # #indxs1 = numpy.array(indxs1[0])
1176 # #indxs2 = numpy.array(indxs2[0])
1195 # #indxs2 = numpy.array(indxs2[0])
1177 # indxs = None
1196 # indxs = None
1178 # #print indxs1 , indxs2
1197 # #print indxs1 , indxs2
1179 # for iv in range(len(indxs2)):
1198 # for iv in range(len(indxs2)):
1180 # indv = numpy.array((indxs1 == indxs2[iv]).nonzero())
1199 # indv = numpy.array((indxs1 == indxs2[iv]).nonzero())
1181 # #print len(indxs2), indv
1200 # #print len(indxs2), indv
1182 # if len(indv[0]) > 0 :
1201 # if len(indv[0]) > 0 :
1183 # indxs = numpy.concatenate((indxs,indxs2[iv]), axis=None)
1202 # indxs = numpy.concatenate((indxs,indxs2[iv]), axis=None)
1184 # # print indxs
1203 # # print indxs
1185 # indxs = indxs[1:]
1204 # indxs = indxs[1:]
1186 # #print(indxs, len(indxs))
1205 # #print(indxs, len(indxs))
1187 # if len(indxs) < 4 :
1206 # if len(indxs) < 4 :
1188 # array[ii,:,:] = 0.
1207 # array[ii,:,:] = 0.
1189 # return
1208 # return
1190 #
1209 #
1191 # xpos = numpy.mod(indxs ,num_hei)
1210 # xpos = numpy.mod(indxs ,num_hei)
1192 # ypos = (indxs / num_hei)
1211 # ypos = (indxs / num_hei)
1193 # sx = numpy.argsort(xpos) # Ordering respect to "x" (time)
1212 # sx = numpy.argsort(xpos) # Ordering respect to "x" (time)
1194 # #print sx
1213 # #print sx
1195 # xpos = xpos[sx]
1214 # xpos = xpos[sx]
1196 # ypos = ypos[sx]
1215 # ypos = ypos[sx]
1197 #
1216 #
1198 # # *********************************** Cleaning isolated points **********************************
1217 # # *********************************** Cleaning isolated points **********************************
1199 # ic = 0
1218 # ic = 0
1200 # while True :
1219 # while True :
1201 # r = numpy.sqrt(list(numpy.power((xpos[ic]-xpos),2)+ numpy.power((ypos[ic]-ypos),2)))
1220 # r = numpy.sqrt(list(numpy.power((xpos[ic]-xpos),2)+ numpy.power((ypos[ic]-ypos),2)))
1202 # #no_coh = WHERE(FINITE(r) AND (r LE rth),cno_coh)
1221 # #no_coh = WHERE(FINITE(r) AND (r LE rth),cno_coh)
1203 # #plt.plot(r)
1222 # #plt.plot(r)
1204 # #plt.show()
1223 # #plt.show()
1205 # no_coh1 = (numpy.isfinite(r)==True).nonzero()
1224 # no_coh1 = (numpy.isfinite(r)==True).nonzero()
1206 # no_coh2 = (r <= rth).nonzero()
1225 # no_coh2 = (r <= rth).nonzero()
1207 # #print r, no_coh1, no_coh2
1226 # #print r, no_coh1, no_coh2
1208 # no_coh1 = numpy.array(no_coh1[0])
1227 # no_coh1 = numpy.array(no_coh1[0])
1209 # no_coh2 = numpy.array(no_coh2[0])
1228 # no_coh2 = numpy.array(no_coh2[0])
1210 # no_coh = None
1229 # no_coh = None
1211 # #print valid1 , valid2
1230 # #print valid1 , valid2
1212 # for iv in range(len(no_coh2)):
1231 # for iv in range(len(no_coh2)):
1213 # indv = numpy.array((no_coh1 == no_coh2[iv]).nonzero())
1232 # indv = numpy.array((no_coh1 == no_coh2[iv]).nonzero())
1214 # if len(indv[0]) > 0 :
1233 # if len(indv[0]) > 0 :
1215 # no_coh = numpy.concatenate((no_coh,no_coh2[iv]), axis=None)
1234 # no_coh = numpy.concatenate((no_coh,no_coh2[iv]), axis=None)
1216 # no_coh = no_coh[1:]
1235 # no_coh = no_coh[1:]
1217 # #print len(no_coh), no_coh
1236 # #print len(no_coh), no_coh
1218 # if len(no_coh) < 4 :
1237 # if len(no_coh) < 4 :
1219 # #print xpos[ic], ypos[ic], ic
1238 # #print xpos[ic], ypos[ic], ic
1220 # # plt.plot(r)
1239 # # plt.plot(r)
1221 # # plt.show()
1240 # # plt.show()
1222 # xpos[ic] = numpy.nan
1241 # xpos[ic] = numpy.nan
1223 # ypos[ic] = numpy.nan
1242 # ypos[ic] = numpy.nan
1224 #
1243 #
1225 # ic = ic + 1
1244 # ic = ic + 1
1226 # if (ic == len(indxs)) :
1245 # if (ic == len(indxs)) :
1227 # break
1246 # break
1228 # #print( xpos, ypos)
1247 # #print( xpos, ypos)
1229 #
1248 #
1230 # indxs = (numpy.isfinite(list(xpos))==True).nonzero()
1249 # indxs = (numpy.isfinite(list(xpos))==True).nonzero()
1231 # #print indxs[0]
1250 # #print indxs[0]
1232 # if len(indxs[0]) < 4 :
1251 # if len(indxs[0]) < 4 :
1233 # array[ii,:,:] = 0.
1252 # array[ii,:,:] = 0.
1234 # return
1253 # return
1235 #
1254 #
1236 # xpos = xpos[indxs[0]]
1255 # xpos = xpos[indxs[0]]
1237 # ypos = ypos[indxs[0]]
1256 # ypos = ypos[indxs[0]]
1238 # for i in range(0,len(ypos)):
1257 # for i in range(0,len(ypos)):
1239 # ypos[i]=int(ypos[i])
1258 # ypos[i]=int(ypos[i])
1240 # junk = tmp
1259 # junk = tmp
1241 # tmp = junk*0.0
1260 # tmp = junk*0.0
1242 #
1261 #
1243 # tmp[list(xpos + (ypos*num_hei))] = junk[list(xpos + (ypos*num_hei))]
1262 # tmp[list(xpos + (ypos*num_hei))] = junk[list(xpos + (ypos*num_hei))]
1244 # array[ii,:,:] = numpy.reshape(tmp,(num_prof,num_hei))
1263 # array[ii,:,:] = numpy.reshape(tmp,(num_prof,num_hei))
1245 #
1264 #
1246 # #print array.shape
1265 # #print array.shape
1247 # #tmp = numpy.reshape(tmp,(num_prof,num_hei))
1266 # #tmp = numpy.reshape(tmp,(num_prof,num_hei))
1248 # #print tmp.shape
1267 # #print tmp.shape
1249 #
1268 #
1250 # # fig = plt.figure(figsize=(6,5))
1269 # # fig = plt.figure(figsize=(6,5))
1251 # # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
1270 # # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
1252 # # ax = fig.add_axes([left, bottom, width, height])
1271 # # ax = fig.add_axes([left, bottom, width, height])
1253 # # x = range(num_prof)
1272 # # x = range(num_prof)
1254 # # y = range(num_hei)
1273 # # y = range(num_hei)
1255 # # cp = ax.contour(y,x,array[ii,:,:])
1274 # # cp = ax.contour(y,x,array[ii,:,:])
1256 # # ax.clabel(cp, inline=True,fontsize=10)
1275 # # ax.clabel(cp, inline=True,fontsize=10)
1257 # # plt.show()
1276 # # plt.show()
1258 # return array
1277 # return array
1259 #
1278 #
1260
1279
1261 class IntegrationFaradaySpectra(Operation):
1280 class IntegrationFaradaySpectra(Operation):
1262
1281
1263 __profIndex = 0
1282 __profIndex = 0
1264 __withOverapping = False
1283 __withOverapping = False
1265
1284
1266 __byTime = False
1285 __byTime = False
1267 __initime = None
1286 __initime = None
1268 __lastdatatime = None
1287 __lastdatatime = None
1269 __integrationtime = None
1288 __integrationtime = None
1270
1289
1271 __buffer_spc = None
1290 __buffer_spc = None
1272 __buffer_cspc = None
1291 __buffer_cspc = None
1273 __buffer_dc = None
1292 __buffer_dc = None
1274
1293
1275 __dataReady = False
1294 __dataReady = False
1276
1295
1277 __timeInterval = None
1296 __timeInterval = None
1278 n_ints = None #matriz de numero de integracions (CH,HEI)
1297 n_ints = None #matriz de numero de integracions (CH,HEI)
1279 n = None
1298 n = None
1280 minHei_ind = None
1299 minHei_ind = None
1281 maxHei_ind = None
1300 maxHei_ind = None
1282 navg = 1.0
1301 navg = 1.0
1283 factor = 0.0
1302 factor = 0.0
1284 dataoutliers = None # (CHANNELS, HEIGHTS)
1303 dataoutliers = None # (CHANNELS, HEIGHTS)
1285
1304
1286 def __init__(self):
1305 def __init__(self):
1287
1306
1288 Operation.__init__(self)
1307 Operation.__init__(self)
1289
1308
1290 def setup(self, dataOut,n=None, timeInterval=None, overlapping=False, DPL=None, minHei=None, maxHei=None, avg=1,factor=0.75):
1309 def setup(self, dataOut,n=None, timeInterval=None, overlapping=False, DPL=None, minHei=None, maxHei=None, avg=1,factor=0.75):
1291 """
1310 """
1292 Set the parameters of the integration class.
1311 Set the parameters of the integration class.
1293
1312
1294 Inputs:
1313 Inputs:
1295
1314
1296 n : Number of coherent integrations
1315 n : Number of coherent integrations
1297 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1316 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1298 overlapping :
1317 overlapping :
1299
1318
1300 """
1319 """
1301
1320
1302 self.__initime = None
1321 self.__initime = None
1303 self.__lastdatatime = 0
1322 self.__lastdatatime = 0
1304
1323
1305 self.__buffer_spc = []
1324 self.__buffer_spc = []
1306 self.__buffer_cspc = []
1325 self.__buffer_cspc = []
1307 self.__buffer_dc = 0
1326 self.__buffer_dc = 0
1308
1327
1309 self.__profIndex = 0
1328 self.__profIndex = 0
1310 self.__dataReady = False
1329 self.__dataReady = False
1311 self.__byTime = False
1330 self.__byTime = False
1312
1331
1313 self.factor = factor
1332 self.factor = factor
1314 self.navg = avg
1333 self.navg = avg
1315 #self.ByLags = dataOut.ByLags ###REDEFINIR
1334 #self.ByLags = dataOut.ByLags ###REDEFINIR
1316 self.ByLags = False
1335 self.ByLags = False
1317 self.maxProfilesInt = 0
1336 self.maxProfilesInt = 0
1318 self.__nChannels = dataOut.nChannels
1337 self.__nChannels = dataOut.nChannels
1319 if DPL != None:
1338 if DPL != None:
1320 self.DPL=DPL
1339 self.DPL=DPL
1321 else:
1340 else:
1322 #self.DPL=dataOut.DPL ###REDEFINIR
1341 #self.DPL=dataOut.DPL ###REDEFINIR
1323 self.DPL=0
1342 self.DPL=0
1324
1343
1325 if n is None and timeInterval is None:
1344 if n is None and timeInterval is None:
1326 raise ValueError("n or timeInterval should be specified ...")
1345 raise ValueError("n or timeInterval should be specified ...")
1327
1346
1328 if n is not None:
1347 if n is not None:
1329 self.n = int(n)
1348 self.n = int(n)
1330 else:
1349 else:
1331 self.__integrationtime = int(timeInterval)
1350 self.__integrationtime = int(timeInterval)
1332 self.n = None
1351 self.n = None
1333 self.__byTime = True
1352 self.__byTime = True
1334
1353
1354
1335 if minHei == None:
1355 if minHei == None:
1336 minHei = self.dataOut.heightList[0]
1356 minHei = self.dataOut.heightList[0]
1337
1357
1338 if maxHei == None:
1358 if maxHei == None:
1339 maxHei = self.dataOut.heightList[-1]
1359 maxHei = self.dataOut.heightList[-1]
1340
1360
1341 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
1361 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
1342 print('minHei: %.2f is out of the heights range' % (minHei))
1362 print('minHei: %.2f is out of the heights range' % (minHei))
1343 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
1363 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
1344 minHei = self.dataOut.heightList[0]
1364 minHei = self.dataOut.heightList[0]
1345
1365
1346 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
1366 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
1347 print('maxHei: %.2f is out of the heights range' % (maxHei))
1367 print('maxHei: %.2f is out of the heights range' % (maxHei))
1348 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
1368 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
1349 maxHei = self.dataOut.heightList[-1]
1369 maxHei = self.dataOut.heightList[-1]
1350
1370
1351 ind_list1 = numpy.where(self.dataOut.heightList >= minHei)
1371 ind_list1 = numpy.where(self.dataOut.heightList >= minHei)
1352 ind_list2 = numpy.where(self.dataOut.heightList <= maxHei)
1372 ind_list2 = numpy.where(self.dataOut.heightList <= maxHei)
1353 self.minHei_ind = ind_list1[0][0]
1373 self.minHei_ind = ind_list1[0][0]
1354 self.maxHei_ind = ind_list2[0][-1]
1374 self.maxHei_ind = ind_list2[0][-1]
1355 #print("setup rem sats done")
1375
1356
1376
1357 def putData(self, data_spc, data_cspc, data_dc):
1377 def putData(self, data_spc, data_cspc, data_dc):
1358 """
1378 """
1359 Add a profile to the __buffer_spc and increase in one the __profileIndex
1379 Add a profile to the __buffer_spc and increase in one the __profileIndex
1360
1380
1361 """
1381 """
1362
1382
1363 self.__buffer_spc.append(data_spc)
1383 self.__buffer_spc.append(data_spc)
1364
1384
1365 if self.__nChannels < 2:
1385 if self.__nChannels < 2:
1366 self.__buffer_cspc = None
1386 self.__buffer_cspc = None
1367 else:
1387 else:
1368 self.__buffer_cspc.append(data_cspc)
1388 self.__buffer_cspc.append(data_cspc)
1369
1389
1370 if data_dc is None:
1390 if data_dc is None:
1371 self.__buffer_dc = None
1391 self.__buffer_dc = None
1372 else:
1392 else:
1373 self.__buffer_dc += data_dc
1393 self.__buffer_dc += data_dc
1374
1394
1375 self.__profIndex += 1
1395 self.__profIndex += 1
1376
1396
1377 return
1397 return
1378
1398
1379 def hildebrand_sekhon_Integration(self,sortdata,navg, factor):
1399 def hildebrand_sekhon_Integration(self,sortdata,navg, factor):
1380 #data debe estar ordenado
1400 #data debe estar ordenado
1381 #sortdata = numpy.sort(data, axis=None)
1401 #sortdata = numpy.sort(data, axis=None)
1382 #sortID=data.argsort()
1402 #sortID=data.argsort()
1383 lenOfData = len(sortdata)
1403 lenOfData = len(sortdata)
1384 nums_min = lenOfData*factor
1404 nums_min = lenOfData*factor
1385 if nums_min <= 5:
1405 if nums_min <= 5:
1386 nums_min = 5
1406 nums_min = 5
1387 sump = 0.
1407 sump = 0.
1388 sumq = 0.
1408 sumq = 0.
1389 j = 0
1409 j = 0
1390 cont = 1
1410 cont = 1
1391 while((cont == 1)and(j < lenOfData)):
1411 while((cont == 1)and(j < lenOfData)):
1392 sump += sortdata[j]
1412 sump += sortdata[j]
1393 sumq += sortdata[j]**2
1413 sumq += sortdata[j]**2
1394 if j > nums_min:
1414 if j > nums_min:
1395 rtest = float(j)/(j-1) + 1.0/navg
1415 rtest = float(j)/(j-1) + 1.0/navg
1396 if ((sumq*j) > (rtest*sump**2)):
1416 if ((sumq*j) > (rtest*sump**2)):
1397 j = j - 1
1417 j = j - 1
1398 sump = sump - sortdata[j]
1418 sump = sump - sortdata[j]
1399 sumq = sumq - sortdata[j]**2
1419 sumq = sumq - sortdata[j]**2
1400 cont = 0
1420 cont = 0
1401 j += 1
1421 j += 1
1402 #lnoise = sump / j
1422 #lnoise = sump / j
1403 #print("H S done")
1423 #print("H S done")
1404 #return j,sortID
1424 #return j,sortID
1405 return j
1425 return j
1406
1426
1407
1427
1408 def pushData(self):
1428 def pushData(self):
1409 """
1429 """
1410 Return the sum of the last profiles and the profiles used in the sum.
1430 Return the sum of the last profiles and the profiles used in the sum.
1411
1431
1412 Affected:
1432 Affected:
1413
1433
1414 self.__profileIndex
1434 self.__profileIndex
1415
1435
1416 """
1436 """
1417 bufferH=None
1437 bufferH=None
1418 buffer=None
1438 buffer=None
1419 buffer1=None
1439 buffer1=None
1420 buffer_cspc=None
1440 buffer_cspc=None
1421 #print("aes: ", self.__buffer_cspc)
1441 #print("aes: ", self.__buffer_cspc)
1422 self.__buffer_spc=numpy.array(self.__buffer_spc)
1442 self.__buffer_spc=numpy.array(self.__buffer_spc)
1423 if self.__nChannels > 1 :
1443 if self.__nChannels > 1 :
1424 self.__buffer_cspc=numpy.array(self.__buffer_cspc)
1444 self.__buffer_cspc=numpy.array(self.__buffer_cspc)
1425
1445
1426 #print("FREQ_DC",self.__buffer_spc.shape,self.__buffer_cspc.shape)
1446 #print("FREQ_DC",self.__buffer_spc.shape,self.__buffer_cspc.shape)
1427
1447
1428 freq_dc = int(self.__buffer_spc.shape[2] / 2)
1448 freq_dc = int(self.__buffer_spc.shape[2] / 2)
1429 #print("FREQ_DC",freq_dc,self.__buffer_spc.shape,self.nHeights)
1449 #print("FREQ_DC",freq_dc,self.__buffer_spc.shape,self.nHeights)
1430
1450
1431 self.dataOutliers = numpy.zeros((self.nChannels,self.nHeights)) # --> almacen de outliers
1451 self.dataOutliers = numpy.zeros((self.nChannels,self.nHeights)) # --> almacen de outliers
1432
1452
1433 for k in range(self.minHei_ind,self.maxHei_ind):
1453 for k in range(self.minHei_ind,self.maxHei_ind):
1434 if self.__nChannels > 1:
1454 if self.__nChannels > 1:
1435 buffer_cspc=numpy.copy(self.__buffer_cspc[:,:,:,k])
1455 buffer_cspc=numpy.copy(self.__buffer_cspc[:,:,:,k])
1436
1456
1437 outliers_IDs_cspc=[]
1457 outliers_IDs_cspc=[]
1438 cspc_outliers_exist=False
1458 cspc_outliers_exist=False
1439 for i in range(self.nChannels):#dataOut.nChannels):
1459 for i in range(self.nChannels):#dataOut.nChannels):
1440
1460
1441 buffer1=numpy.copy(self.__buffer_spc[:,i,:,k])
1461 buffer1=numpy.copy(self.__buffer_spc[:,i,:,k])
1442 indexes=[]
1462 indexes=[]
1443 #sortIDs=[]
1463 #sortIDs=[]
1444 outliers_IDs=[]
1464 outliers_IDs=[]
1445
1465
1446 for j in range(self.nProfiles): #frecuencias en el tiempo
1466 for j in range(self.nProfiles): #frecuencias en el tiempo
1447 # if i==0 and j==freq_dc: #NOT CONSIDERING DC PROFILE AT CHANNEL 0
1467 # if i==0 and j==freq_dc: #NOT CONSIDERING DC PROFILE AT CHANNEL 0
1448 # continue
1468 # continue
1449 # if i==1 and j==0: #NOT CONSIDERING DC PROFILE AT CHANNEL 1
1469 # if i==1 and j==0: #NOT CONSIDERING DC PROFILE AT CHANNEL 1
1450 # continue
1470 # continue
1451 buffer=buffer1[:,j]
1471 buffer=buffer1[:,j]
1452 sortdata = numpy.sort(buffer, axis=None)
1472 sortdata = numpy.sort(buffer, axis=None)
1453
1473
1454 sortID=buffer.argsort()
1474 sortID=buffer.argsort()
1455 index = _noise.hildebrand_sekhon2(sortdata,self.navg)
1475 index = _noise.hildebrand_sekhon2(sortdata,self.navg)
1456
1476
1457 #index,sortID=self.hildebrand_sekhon_Integration(buffer,1,self.factor)
1477 #index,sortID=self.hildebrand_sekhon_Integration(buffer,1,self.factor)
1458
1478
1459 # fig,ax = plt.subplots()
1479 # fig,ax = plt.subplots()
1460 # ax.set_title(str(k)+" "+str(j))
1480 # ax.set_title(str(k)+" "+str(j))
1461 # x=range(len(sortdata))
1481 # x=range(len(sortdata))
1462 # ax.scatter(x,sortdata)
1482 # ax.scatter(x,sortdata)
1463 # ax.axvline(index)
1483 # ax.axvline(index)
1464 # plt.show()
1484 # plt.show()
1465
1485
1466 indexes.append(index)
1486 indexes.append(index)
1467 #sortIDs.append(sortID)
1487 #sortIDs.append(sortID)
1468 outliers_IDs=numpy.append(outliers_IDs,sortID[index:])
1488 outliers_IDs=numpy.append(outliers_IDs,sortID[index:])
1469
1489
1470 #print("Outliers: ",outliers_IDs)
1490 #print("Outliers: ",outliers_IDs)
1471 outliers_IDs=numpy.array(outliers_IDs)
1491 outliers_IDs=numpy.array(outliers_IDs)
1472 outliers_IDs=outliers_IDs.ravel()
1492 outliers_IDs=outliers_IDs.ravel()
1473 outliers_IDs=numpy.unique(outliers_IDs)
1493 outliers_IDs=numpy.unique(outliers_IDs)
1474 outliers_IDs=outliers_IDs.astype(numpy.dtype('int64'))
1494 outliers_IDs=outliers_IDs.astype(numpy.dtype('int64'))
1475 indexes=numpy.array(indexes)
1495 indexes=numpy.array(indexes)
1476 indexmin=numpy.min(indexes)
1496 indexmin=numpy.min(indexes)
1477
1497
1478
1498
1479 #print(indexmin,buffer1.shape[0], k)
1499 #print(indexmin,buffer1.shape[0], k)
1480
1500
1481 # fig,ax = plt.subplots()
1501 # fig,ax = plt.subplots()
1482 # ax.plot(sortdata)
1502 # ax.plot(sortdata)
1483 # ax2 = ax.twinx()
1503 # ax2 = ax.twinx()
1484 # x=range(len(indexes))
1504 # x=range(len(indexes))
1485 # #plt.scatter(x,indexes)
1505 # #plt.scatter(x,indexes)
1486 # ax2.scatter(x,indexes)
1506 # ax2.scatter(x,indexes)
1487 # plt.show()
1507 # plt.show()
1488
1508
1489 if indexmin != buffer1.shape[0]:
1509 if indexmin != buffer1.shape[0]:
1490 if self.__nChannels > 1:
1510 if self.__nChannels > 1:
1491 cspc_outliers_exist= True
1511 cspc_outliers_exist= True
1492
1512
1493 lt=outliers_IDs
1513 lt=outliers_IDs
1494 #avg=numpy.mean(buffer1[[t for t in range(buffer1.shape[0]) if t not in lt],:],axis=0)
1514 #avg=numpy.mean(buffer1[[t for t in range(buffer1.shape[0]) if t not in lt],:],axis=0)
1495
1515
1496 for p in list(outliers_IDs):
1516 for p in list(outliers_IDs):
1497 #buffer1[p,:]=avg
1517 #buffer1[p,:]=avg
1498 buffer1[p,:] = numpy.NaN
1518 buffer1[p,:] = numpy.NaN
1499
1519
1500 self.dataOutliers[i,k] = len(outliers_IDs)
1520 self.dataOutliers[i,k] = len(outliers_IDs)
1501
1521
1502
1522
1503 self.__buffer_spc[:,i,:,k]=numpy.copy(buffer1)
1523 self.__buffer_spc[:,i,:,k]=numpy.copy(buffer1)
1504
1524
1505
1525
1506 if self.__nChannels > 1:
1526 if self.__nChannels > 1:
1507 outliers_IDs_cspc=numpy.append(outliers_IDs_cspc,outliers_IDs)
1527 outliers_IDs_cspc=numpy.append(outliers_IDs_cspc,outliers_IDs)
1508
1528
1509
1529
1510 if self.__nChannels > 1:
1530 if self.__nChannels > 1:
1511 outliers_IDs_cspc=outliers_IDs_cspc.astype(numpy.dtype('int64'))
1531 outliers_IDs_cspc=outliers_IDs_cspc.astype(numpy.dtype('int64'))
1512 if cspc_outliers_exist:
1532 if cspc_outliers_exist:
1513
1533
1514 lt=outliers_IDs_cspc
1534 lt=outliers_IDs_cspc
1515
1535
1516 #avg=numpy.mean(buffer_cspc[[t for t in range(buffer_cspc.shape[0]) if t not in lt],:],axis=0)
1536 #avg=numpy.mean(buffer_cspc[[t for t in range(buffer_cspc.shape[0]) if t not in lt],:],axis=0)
1517 for p in list(outliers_IDs_cspc):
1537 for p in list(outliers_IDs_cspc):
1518 #buffer_cspc[p,:]=avg
1538 #buffer_cspc[p,:]=avg
1519 buffer_cspc[p,:] = numpy.NaN
1539 buffer_cspc[p,:] = numpy.NaN
1520
1540
1521 if self.__nChannels > 1:
1541 if self.__nChannels > 1:
1522 self.__buffer_cspc[:,:,:,k]=numpy.copy(buffer_cspc)
1542 self.__buffer_cspc[:,:,:,k]=numpy.copy(buffer_cspc)
1523
1543
1524
1544
1525
1545
1526
1546
1527 nOutliers = len(outliers_IDs)
1547 nOutliers = len(outliers_IDs)
1528 #print("Outliers n: ",self.dataOutliers,nOutliers)
1548 #print("Outliers n: ",self.dataOutliers,nOutliers)
1529 buffer=None
1549 buffer=None
1530 bufferH=None
1550 bufferH=None
1531 buffer1=None
1551 buffer1=None
1532 buffer_cspc=None
1552 buffer_cspc=None
1533
1553
1534
1554
1535 buffer=None
1555 buffer=None
1536
1556
1537 #data_spc = numpy.sum(self.__buffer_spc,axis=0)
1557 #data_spc = numpy.sum(self.__buffer_spc,axis=0)
1538 data_spc = numpy.nansum(self.__buffer_spc,axis=0)
1558 data_spc = numpy.nansum(self.__buffer_spc,axis=0)
1539 if self.__nChannels > 1:
1559 if self.__nChannels > 1:
1540 #data_cspc = numpy.sum(self.__buffer_cspc,axis=0)
1560 #data_cspc = numpy.sum(self.__buffer_cspc,axis=0)
1541 data_cspc = numpy.nansum(self.__buffer_cspc,axis=0)
1561 data_cspc = numpy.nansum(self.__buffer_cspc,axis=0)
1542 else:
1562 else:
1543 data_cspc = None
1563 data_cspc = None
1544 data_dc = self.__buffer_dc
1564 data_dc = self.__buffer_dc
1545 #(CH, HEIGH)
1565 #(CH, HEIGH)
1546 self.maxProfilesInt = self.__profIndex - 1
1566 self.maxProfilesInt = self.__profIndex - 1
1547 n = self.__profIndex - self.dataOutliers # n becomes a matrix
1567 n = self.__profIndex - self.dataOutliers # n becomes a matrix
1548
1568
1549 self.__buffer_spc = []
1569 self.__buffer_spc = []
1550 self.__buffer_cspc = []
1570 self.__buffer_cspc = []
1551 self.__buffer_dc = 0
1571 self.__buffer_dc = 0
1552 self.__profIndex = 0
1572 self.__profIndex = 0
1553 #print("cleaned ",data_cspc)
1573 #print("cleaned ",data_cspc)
1554 return data_spc, data_cspc, data_dc, n
1574 return data_spc, data_cspc, data_dc, n
1555
1575
1556 def byProfiles(self, *args):
1576 def byProfiles(self, *args):
1557
1577
1558 self.__dataReady = False
1578 self.__dataReady = False
1559 avgdata_spc = None
1579 avgdata_spc = None
1560 avgdata_cspc = None
1580 avgdata_cspc = None
1561 avgdata_dc = None
1581 avgdata_dc = None
1562
1582
1563 self.putData(*args)
1583 self.putData(*args)
1564
1584
1565 if self.__profIndex == self.n:
1585 if self.__profIndex == self.n:
1566
1586
1567 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1587 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1568 self.n_ints = n
1588 self.n_ints = n
1569 self.__dataReady = True
1589 self.__dataReady = True
1570
1590
1571 return avgdata_spc, avgdata_cspc, avgdata_dc
1591 return avgdata_spc, avgdata_cspc, avgdata_dc
1572
1592
1573 def byTime(self, datatime, *args):
1593 def byTime(self, datatime, *args):
1574
1594
1575 self.__dataReady = False
1595 self.__dataReady = False
1576 avgdata_spc = None
1596 avgdata_spc = None
1577 avgdata_cspc = None
1597 avgdata_cspc = None
1578 avgdata_dc = None
1598 avgdata_dc = None
1579
1599
1580 self.putData(*args)
1600 self.putData(*args)
1581
1601
1582 if (datatime - self.__initime) >= self.__integrationtime:
1602 if (datatime - self.__initime) >= self.__integrationtime:
1583 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1603 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1584 self.n_ints = n
1604 self.n_ints = n
1585 self.__dataReady = True
1605 self.__dataReady = True
1586
1606
1587 return avgdata_spc, avgdata_cspc, avgdata_dc
1607 return avgdata_spc, avgdata_cspc, avgdata_dc
1588
1608
1589 def integrate(self, datatime, *args):
1609 def integrate(self, datatime, *args):
1590
1610
1591 if self.__profIndex == 0:
1611 if self.__profIndex == 0:
1592 self.__initime = datatime
1612 self.__initime = datatime
1593
1613
1594 if self.__byTime:
1614 if self.__byTime:
1595 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
1615 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
1596 datatime, *args)
1616 datatime, *args)
1597 else:
1617 else:
1598 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1618 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1599
1619
1600 if not self.__dataReady:
1620 if not self.__dataReady:
1601 return None, None, None, None
1621 return None, None, None, None
1602
1622
1603 #print("integrate", avgdata_cspc)
1623 #print("integrate", avgdata_cspc)
1604 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
1624 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
1605
1625
1606 def run(self, dataOut, n=None, DPL = None,timeInterval=None, overlapping=False, minHei=None, maxHei=None, avg=1, factor=0.75):
1626 def run(self, dataOut, n=None, DPL = None,timeInterval=None, overlapping=False, minHei=None, maxHei=None, avg=1, factor=0.75):
1607 self.dataOut = dataOut
1627 self.dataOut = dataOut
1608 if n == 1:
1628 if n == 1:
1609 return self.dataOut
1629 return self.dataOut
1610
1630 self.dataOut.processingHeaderObj.timeIncohInt = timeInterval
1611 #print("nchannels", self.dataOut.nChannels)
1631 #print("nchannels", self.dataOut.nChannels)
1612 if self.dataOut.nChannels == 1:
1632 if self.dataOut.nChannels == 1:
1613 self.dataOut.data_cspc = None #si es un solo canal no vale la pena acumular DATOS
1633 self.dataOut.data_cspc = None #si es un solo canal no vale la pena acumular DATOS
1614 #print("IN spc:", self.dataOut.data_spc.shape, self.dataOut.data_cspc)
1634 #print("IN spc:", self.dataOut.data_spc.shape, self.dataOut.data_cspc)
1615 if not self.isConfig:
1635 if not self.isConfig:
1616 self.setup(self.dataOut, n, timeInterval, overlapping,DPL ,minHei, maxHei, avg, factor)
1636 self.setup(self.dataOut, n, timeInterval, overlapping,DPL ,minHei, maxHei, avg, factor)
1617 self.isConfig = True
1637 self.isConfig = True
1618
1638
1619 if not self.ByLags:
1639 if not self.ByLags:
1620 self.nProfiles=self.dataOut.nProfiles
1640 self.nProfiles=self.dataOut.nProfiles
1621 self.nChannels=self.dataOut.nChannels
1641 self.nChannels=self.dataOut.nChannels
1622 self.nHeights=self.dataOut.nHeights
1642 self.nHeights=self.dataOut.nHeights
1623 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
1643 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
1624 self.dataOut.data_spc,
1644 self.dataOut.data_spc,
1625 self.dataOut.data_cspc,
1645 self.dataOut.data_cspc,
1626 self.dataOut.data_dc)
1646 self.dataOut.data_dc)
1627 else:
1647 else:
1628 self.nProfiles=self.dataOut.nProfiles
1648 self.nProfiles=self.dataOut.nProfiles
1629 self.nChannels=self.dataOut.nChannels
1649 self.nChannels=self.dataOut.nChannels
1630 self.nHeights=self.dataOut.nHeights
1650 self.nHeights=self.dataOut.nHeights
1631 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
1651 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
1632 self.dataOut.dataLag_spc,
1652 self.dataOut.dataLag_spc,
1633 self.dataOut.dataLag_cspc,
1653 self.dataOut.dataLag_cspc,
1634 self.dataOut.dataLag_dc)
1654 self.dataOut.dataLag_dc)
1635 self.dataOut.flagNoData = True
1655 self.dataOut.flagNoData = True
1636 if self.__dataReady:
1656 if self.__dataReady:
1637
1657
1638 if not self.ByLags:
1658 if not self.ByLags:
1639 if self.nChannels == 1:
1659 if self.nChannels == 1:
1640 #print("f int", avgdata_spc.shape)
1660 #print("f int", avgdata_spc.shape)
1641 self.dataOut.data_spc = avgdata_spc
1661 self.dataOut.data_spc = avgdata_spc
1642 self.dataOut.data_cspc = None
1662 self.dataOut.data_cspc = None
1643 else:
1663 else:
1644 self.dataOut.data_spc = numpy.squeeze(avgdata_spc)
1664 self.dataOut.data_spc = numpy.squeeze(avgdata_spc)
1645 self.dataOut.data_cspc = numpy.squeeze(avgdata_cspc)
1665 self.dataOut.data_cspc = numpy.squeeze(avgdata_cspc)
1646 self.dataOut.data_dc = avgdata_dc
1666 self.dataOut.data_dc = avgdata_dc
1647 self.dataOut.data_outlier = self.dataOutliers
1667 self.dataOut.data_outlier = self.dataOutliers
1648
1668
1649 else:
1669 else:
1650 self.dataOut.dataLag_spc = avgdata_spc
1670 self.dataOut.dataLag_spc = avgdata_spc
1651 self.dataOut.dataLag_cspc = avgdata_cspc
1671 self.dataOut.dataLag_cspc = avgdata_cspc
1652 self.dataOut.dataLag_dc = avgdata_dc
1672 self.dataOut.dataLag_dc = avgdata_dc
1653
1673
1654 self.dataOut.data_spc=self.dataOut.dataLag_spc[:,:,:,self.dataOut.LagPlot]
1674 self.dataOut.data_spc=self.dataOut.dataLag_spc[:,:,:,self.dataOut.LagPlot]
1655 self.dataOut.data_cspc=self.dataOut.dataLag_cspc[:,:,:,self.dataOut.LagPlot]
1675 self.dataOut.data_cspc=self.dataOut.dataLag_cspc[:,:,:,self.dataOut.LagPlot]
1656 self.dataOut.data_dc=self.dataOut.dataLag_dc[:,:,self.dataOut.LagPlot]
1676 self.dataOut.data_dc=self.dataOut.dataLag_dc[:,:,self.dataOut.LagPlot]
1657
1677
1658
1678
1659 self.dataOut.nIncohInt *= self.n_ints
1679 self.dataOut.nIncohInt *= self.n_ints
1660 #print("maxProfilesInt: ",self.maxProfilesInt)
1680 #print("maxProfilesInt: ",self.maxProfilesInt)
1661
1681
1662 self.dataOut.utctime = avgdatatime
1682 self.dataOut.utctime = avgdatatime
1663 self.dataOut.flagNoData = False
1683 self.dataOut.flagNoData = False
1684
1685 # #update Processing Header:
1686 # self.dataOut.processingHeaderObj.nIncohInt =
1687 # self.dataOut.processingHeaderObj.nFFTPoints = self.dataOut.nFFTPoints
1688
1664 #print("Faraday Integration DONE...", self.dataOut.data_cspc)
1689 #print("Faraday Integration DONE...", self.dataOut.data_cspc)
1665 #print(self.dataOut.flagNoData)
1690 #print(self.dataOut.flagNoData)
1666 return self.dataOut
1691 return self.dataOut
1667
1692
1668
1693
1669
1694
1670 class removeInterference(Operation):
1695 class removeInterference(Operation):
1671
1696
1672 def removeInterference3(self, min_hei = None, max_hei = None):
1697 def removeInterference3(self, min_hei = None, max_hei = None):
1673
1698
1674 jspectra = self.dataOut.data_spc
1699 jspectra = self.dataOut.data_spc
1675 #jcspectra = self.dataOut.data_cspc
1700 #jcspectra = self.dataOut.data_cspc
1676 jnoise = self.dataOut.getNoise()
1701 jnoise = self.dataOut.getNoise()
1677 num_incoh = self.dataOut.max_nIncohInt
1702 num_incoh = self.dataOut.max_nIncohInt
1678 #print(jspectra.shape)
1703 #print(jspectra.shape)
1679 num_channel, num_prof, num_hei = jspectra.shape
1704 num_channel, num_prof, num_hei = jspectra.shape
1680 minHei = min_hei
1705 minHei = min_hei
1681 maxHei = max_hei
1706 maxHei = max_hei
1682 ########################################################################
1707 ########################################################################
1683 if minHei == None or (minHei < self.dataOut.heightList[0]):
1708 if minHei == None or (minHei < self.dataOut.heightList[0]):
1684 minHei = self.dataOut.heightList[0]
1709 minHei = self.dataOut.heightList[0]
1685
1710
1686 if maxHei == None or (maxHei > self.dataOut.heightList[-1]):
1711 if maxHei == None or (maxHei > self.dataOut.heightList[-1]):
1687 maxHei = self.dataOut.heightList[-1]
1712 maxHei = self.dataOut.heightList[-1]
1688 minIndex = 0
1713 minIndex = 0
1689 maxIndex = 0
1714 maxIndex = 0
1690 heights = self.dataOut.heightList
1715 heights = self.dataOut.heightList
1691
1716
1692 inda = numpy.where(heights >= minHei)
1717 inda = numpy.where(heights >= minHei)
1693 indb = numpy.where(heights <= maxHei)
1718 indb = numpy.where(heights <= maxHei)
1694
1719
1695 try:
1720 try:
1696 minIndex = inda[0][0]
1721 minIndex = inda[0][0]
1697 except:
1722 except:
1698 minIndex = 0
1723 minIndex = 0
1699 try:
1724 try:
1700 maxIndex = indb[0][-1]
1725 maxIndex = indb[0][-1]
1701 except:
1726 except:
1702 maxIndex = len(heights)
1727 maxIndex = len(heights)
1703
1728
1704 if (minIndex < 0) or (minIndex > maxIndex):
1729 if (minIndex < 0) or (minIndex > maxIndex):
1705 raise ValueError("some value in (%d,%d) is not valid" % (
1730 raise ValueError("some value in (%d,%d) is not valid" % (
1706 minIndex, maxIndex))
1731 minIndex, maxIndex))
1707 if (maxIndex >= self.dataOut.nHeights):
1732 if (maxIndex >= self.dataOut.nHeights):
1708 maxIndex = self.dataOut.nHeights - 1
1733 maxIndex = self.dataOut.nHeights - 1
1709
1734
1710 ########################################################################
1735 ########################################################################
1711
1736
1712
1737
1713 #dataOut.max_nIncohInt * dataOut.nCohInt
1738 #dataOut.max_nIncohInt * dataOut.nCohInt
1714 norm = self.dataOut.nIncohInt /self.dataOut.max_nIncohInt
1739 norm = self.dataOut.nIncohInt /self.dataOut.max_nIncohInt
1715 #print(norm.shape)
1740 #print(norm.shape)
1716 # Subrutina de Remocion de la Interferencia
1741 # Subrutina de Remocion de la Interferencia
1717 for ich in range(num_channel):
1742 for ich in range(num_channel):
1718 # Se ordena los espectros segun su potencia (menor a mayor)
1743 # Se ordena los espectros segun su potencia (menor a mayor)
1719 #power = jspectra[ich, mask_prof, :]
1744 #power = jspectra[ich, mask_prof, :]
1720 interf = jspectra[ich, :, minIndex:maxIndex]
1745 interf = jspectra[ich, :, minIndex:maxIndex]
1721 #print(interf.shape)
1746 #print(interf.shape)
1722 inttef = interf.mean(axis=1)
1747 inttef = interf.mean(axis=1)
1723
1748
1724 for hei in range(num_hei):
1749 for hei in range(num_hei):
1725 temp = jspectra[ich,:, hei]
1750 temp = jspectra[ich,:, hei]
1726 temp -= inttef
1751 temp -= inttef
1727 temp += jnoise[ich]*norm[ich,hei]
1752 temp += jnoise[ich]*norm[ich,hei]
1728 jspectra[ich,:, hei] = temp
1753 jspectra[ich,:, hei] = temp
1729
1754
1730 # Guardar Resultados
1755 # Guardar Resultados
1731 self.dataOut.data_spc = jspectra
1756 self.dataOut.data_spc = jspectra
1732 #self.dataOut.data_cspc = jcspectra
1757 #self.dataOut.data_cspc = jcspectra
1733
1758
1734 return 1
1759 return 1
1735
1760
1736 def removeInterference2(self):
1761 def removeInterference2(self):
1737
1762
1738 cspc = self.dataOut.data_cspc
1763 cspc = self.dataOut.data_cspc
1739 spc = self.dataOut.data_spc
1764 spc = self.dataOut.data_spc
1740 Heights = numpy.arange(cspc.shape[2])
1765 Heights = numpy.arange(cspc.shape[2])
1741 realCspc = numpy.abs(cspc)
1766 realCspc = numpy.abs(cspc)
1742
1767
1743 for i in range(cspc.shape[0]):
1768 for i in range(cspc.shape[0]):
1744 LinePower= numpy.sum(realCspc[i], axis=0)
1769 LinePower= numpy.sum(realCspc[i], axis=0)
1745 Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
1770 Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
1746 SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ]
1771 SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ]
1747 InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
1772 InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
1748 InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
1773 InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
1749 InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
1774 InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
1750
1775
1751
1776
1752 InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
1777 InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
1753 #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
1778 #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
1754 if len(InterferenceRange)<int(cspc.shape[1]*0.3):
1779 if len(InterferenceRange)<int(cspc.shape[1]*0.3):
1755 cspc[i,InterferenceRange,:] = numpy.NaN
1780 cspc[i,InterferenceRange,:] = numpy.NaN
1756
1781
1757 self.dataOut.data_cspc = cspc
1782 self.dataOut.data_cspc = cspc
1758
1783
1759 def removeInterference(self, interf = 2, hei_interf = None, nhei_interf = None, offhei_interf = None):
1784 def removeInterference(self, interf = 2, hei_interf = None, nhei_interf = None, offhei_interf = None):
1760
1785
1761 jspectra = self.dataOut.data_spc
1786 jspectra = self.dataOut.data_spc
1762 jcspectra = self.dataOut.data_cspc
1787 jcspectra = self.dataOut.data_cspc
1763 jnoise = self.dataOut.getNoise()
1788 jnoise = self.dataOut.getNoise()
1764 #num_incoh = self.dataOut.nIncohInt
1789 #num_incoh = self.dataOut.nIncohInt
1765 num_incoh = self.dataOut.max_nIncohInt
1790 num_incoh = self.dataOut.max_nIncohInt
1766 #print("spc: ", jspectra.shape, jcspectra)
1791 #print("spc: ", jspectra.shape, jcspectra)
1767 num_channel = jspectra.shape[0]
1792 num_channel = jspectra.shape[0]
1768 num_prof = jspectra.shape[1]
1793 num_prof = jspectra.shape[1]
1769 num_hei = jspectra.shape[2]
1794 num_hei = jspectra.shape[2]
1770
1795
1771 # hei_interf
1796 # hei_interf
1772 if hei_interf is None:
1797 if hei_interf is None:
1773 count_hei = int(num_hei / 2) # a half of total ranges
1798 count_hei = int(num_hei / 2) # a half of total ranges
1774 hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
1799 hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
1775 hei_interf = numpy.asarray(hei_interf)[0]
1800 hei_interf = numpy.asarray(hei_interf)[0]
1776 #print(hei_interf)
1801 #print(hei_interf)
1777 # nhei_interf
1802 # nhei_interf
1778 if (nhei_interf == None):
1803 if (nhei_interf == None):
1779 nhei_interf = 5
1804 nhei_interf = 5
1780 if (nhei_interf < 1):
1805 if (nhei_interf < 1):
1781 nhei_interf = 1
1806 nhei_interf = 1
1782 if (nhei_interf > count_hei):
1807 if (nhei_interf > count_hei):
1783 nhei_interf = count_hei
1808 nhei_interf = count_hei
1784 if (offhei_interf == None):
1809 if (offhei_interf == None):
1785 offhei_interf = 0
1810 offhei_interf = 0
1786
1811
1787 ind_hei = list(range(num_hei))
1812 ind_hei = list(range(num_hei))
1788 # mask_prof = numpy.asarray(range(num_prof - 2)) + 1
1813 # mask_prof = numpy.asarray(range(num_prof - 2)) + 1
1789 # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
1814 # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
1790 mask_prof = numpy.asarray(list(range(num_prof)))
1815 mask_prof = numpy.asarray(list(range(num_prof)))
1791 num_mask_prof = mask_prof.size
1816 num_mask_prof = mask_prof.size
1792 comp_mask_prof = [0, num_prof / 2]
1817 comp_mask_prof = [0, num_prof / 2]
1793
1818
1794 # noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
1819 # noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
1795 if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
1820 if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
1796 jnoise = numpy.nan
1821 jnoise = numpy.nan
1797 noise_exist = jnoise[0] < numpy.Inf
1822 noise_exist = jnoise[0] < numpy.Inf
1798
1823
1799 # Subrutina de Remocion de la Interferencia
1824 # Subrutina de Remocion de la Interferencia
1800 for ich in range(num_channel):
1825 for ich in range(num_channel):
1801 # Se ordena los espectros segun su potencia (menor a mayor)
1826 # Se ordena los espectros segun su potencia (menor a mayor)
1802 power = jspectra[ich, mask_prof, :]
1827 power = jspectra[ich, mask_prof, :]
1803 power = power[:, hei_interf]
1828 power = power[:, hei_interf]
1804 power = power.sum(axis=0)
1829 power = power.sum(axis=0)
1805 psort = power.ravel().argsort()
1830 psort = power.ravel().argsort()
1806 print(hei_interf[psort[list(range(offhei_interf, nhei_interf + offhei_interf))]])
1831 print(hei_interf[psort[list(range(offhei_interf, nhei_interf + offhei_interf))]])
1807 # Se estima la interferencia promedio en los Espectros de Potencia empleando
1832 # Se estima la interferencia promedio en los Espectros de Potencia empleando
1808 junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
1833 junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
1809 offhei_interf, nhei_interf + offhei_interf))]]]
1834 offhei_interf, nhei_interf + offhei_interf))]]]
1810
1835
1811 if noise_exist:
1836 if noise_exist:
1812 # tmp_noise = jnoise[ich] / num_prof
1837 # tmp_noise = jnoise[ich] / num_prof
1813 tmp_noise = jnoise[ich]
1838 tmp_noise = jnoise[ich]
1814 junkspc_interf = junkspc_interf - tmp_noise
1839 junkspc_interf = junkspc_interf - tmp_noise
1815 #junkspc_interf[:,comp_mask_prof] = 0
1840 #junkspc_interf[:,comp_mask_prof] = 0
1816 print(junkspc_interf.shape)
1841 print(junkspc_interf.shape)
1817 jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
1842 jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
1818 jspc_interf = jspc_interf.transpose()
1843 jspc_interf = jspc_interf.transpose()
1819 # Calculando el espectro de interferencia promedio
1844 # Calculando el espectro de interferencia promedio
1820 noiseid = numpy.where(jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
1845 noiseid = numpy.where(jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
1821 noiseid = noiseid[0]
1846 noiseid = noiseid[0]
1822 cnoiseid = noiseid.size
1847 cnoiseid = noiseid.size
1823 interfid = numpy.where(jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
1848 interfid = numpy.where(jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
1824 interfid = interfid[0]
1849 interfid = interfid[0]
1825 cinterfid = interfid.size
1850 cinterfid = interfid.size
1826
1851
1827 if (cnoiseid > 0):
1852 if (cnoiseid > 0):
1828 jspc_interf[noiseid] = 0
1853 jspc_interf[noiseid] = 0
1829 # Expandiendo los perfiles a limpiar
1854 # Expandiendo los perfiles a limpiar
1830 if (cinterfid > 0):
1855 if (cinterfid > 0):
1831 new_interfid = (
1856 new_interfid = (
1832 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
1857 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
1833 new_interfid = numpy.asarray(new_interfid)
1858 new_interfid = numpy.asarray(new_interfid)
1834 new_interfid = {x for x in new_interfid}
1859 new_interfid = {x for x in new_interfid}
1835 new_interfid = numpy.array(list(new_interfid))
1860 new_interfid = numpy.array(list(new_interfid))
1836 new_cinterfid = new_interfid.size
1861 new_cinterfid = new_interfid.size
1837 else:
1862 else:
1838 new_cinterfid = 0
1863 new_cinterfid = 0
1839
1864
1840 for ip in range(new_cinterfid):
1865 for ip in range(new_cinterfid):
1841 ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
1866 ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
1842 jspc_interf[new_interfid[ip]] = junkspc_interf[ind[nhei_interf // 2], new_interfid[ip]]
1867 jspc_interf[new_interfid[ip]] = junkspc_interf[ind[nhei_interf // 2], new_interfid[ip]]
1843
1868
1844 jspectra[ich, :, ind_hei] = jspectra[ich, :,ind_hei] - jspc_interf # Corregir indices
1869 jspectra[ich, :, ind_hei] = jspectra[ich, :,ind_hei] - jspc_interf # Corregir indices
1845
1870
1846 # Removiendo la interferencia del punto de mayor interferencia
1871 # Removiendo la interferencia del punto de mayor interferencia
1847 ListAux = jspc_interf[mask_prof].tolist()
1872 ListAux = jspc_interf[mask_prof].tolist()
1848 maxid = ListAux.index(max(ListAux))
1873 maxid = ListAux.index(max(ListAux))
1849 print(cinterfid)
1874 print(cinterfid)
1850 if cinterfid > 0:
1875 if cinterfid > 0:
1851 for ip in range(cinterfid * (interf == 2) - 1):
1876 for ip in range(cinterfid * (interf == 2) - 1):
1852 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
1877 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
1853 (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
1878 (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
1854 cind = len(ind)
1879 cind = len(ind)
1855
1880
1856 if (cind > 0):
1881 if (cind > 0):
1857 jspectra[ich, interfid[ip], ind] = tmp_noise * \
1882 jspectra[ich, interfid[ip], ind] = tmp_noise * \
1858 (1 + (numpy.random.uniform(cind) - 0.5) /
1883 (1 + (numpy.random.uniform(cind) - 0.5) /
1859 numpy.sqrt(num_incoh))
1884 numpy.sqrt(num_incoh))
1860
1885
1861 ind = numpy.array([-2, -1, 1, 2])
1886 ind = numpy.array([-2, -1, 1, 2])
1862 xx = numpy.zeros([4, 4])
1887 xx = numpy.zeros([4, 4])
1863
1888
1864 for id1 in range(4):
1889 for id1 in range(4):
1865 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1890 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1866 xx_inv = numpy.linalg.inv(xx)
1891 xx_inv = numpy.linalg.inv(xx)
1867 xx = xx_inv[:, 0]
1892 xx = xx_inv[:, 0]
1868 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1893 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1869 yy = jspectra[ich, mask_prof[ind], :]
1894 yy = jspectra[ich, mask_prof[ind], :]
1870 jspectra[ich, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
1895 jspectra[ich, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
1871
1896
1872 indAux = (jspectra[ich, :, :] < tmp_noise *
1897 indAux = (jspectra[ich, :, :] < tmp_noise *
1873 (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
1898 (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
1874 print(indAux)
1899 print(indAux)
1875 jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
1900 jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
1876 (1 - 1 / numpy.sqrt(num_incoh))
1901 (1 - 1 / numpy.sqrt(num_incoh))
1877
1902
1878 # Remocion de Interferencia en el Cross Spectra
1903 # Remocion de Interferencia en el Cross Spectra
1879 if jcspectra is None:
1904 if jcspectra is None:
1880 return jspectra, jcspectra
1905 return jspectra, jcspectra
1881 num_pairs = int(jcspectra.size / (num_prof * num_hei))
1906 num_pairs = int(jcspectra.size / (num_prof * num_hei))
1882 jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
1907 jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
1883
1908
1884 for ip in range(num_pairs):
1909 for ip in range(num_pairs):
1885
1910
1886 #-------------------------------------------
1911 #-------------------------------------------
1887
1912
1888 cspower = numpy.abs(jcspectra[ip, mask_prof, :])
1913 cspower = numpy.abs(jcspectra[ip, mask_prof, :])
1889 cspower = cspower[:, hei_interf]
1914 cspower = cspower[:, hei_interf]
1890 cspower = cspower.sum(axis=0)
1915 cspower = cspower.sum(axis=0)
1891
1916
1892 cspsort = cspower.ravel().argsort()
1917 cspsort = cspower.ravel().argsort()
1893 junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
1918 junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
1894 offhei_interf, nhei_interf + offhei_interf))]]]
1919 offhei_interf, nhei_interf + offhei_interf))]]]
1895 junkcspc_interf = junkcspc_interf.transpose()
1920 junkcspc_interf = junkcspc_interf.transpose()
1896 jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
1921 jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
1897
1922
1898 ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
1923 ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
1899
1924
1900 median_real = int(numpy.median(numpy.real(
1925 median_real = int(numpy.median(numpy.real(
1901 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1926 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1902 median_imag = int(numpy.median(numpy.imag(
1927 median_imag = int(numpy.median(numpy.imag(
1903 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1928 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1904 comp_mask_prof = [int(e) for e in comp_mask_prof]
1929 comp_mask_prof = [int(e) for e in comp_mask_prof]
1905 junkcspc_interf[comp_mask_prof, :] = numpy.complex(
1930 junkcspc_interf[comp_mask_prof, :] = numpy.complex(
1906 median_real, median_imag)
1931 median_real, median_imag)
1907
1932
1908 for iprof in range(num_prof):
1933 for iprof in range(num_prof):
1909 ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
1934 ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
1910 jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf // 2]]
1935 jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf // 2]]
1911
1936
1912 # Removiendo la Interferencia
1937 # Removiendo la Interferencia
1913 jcspectra[ip, :, ind_hei] = jcspectra[ip,
1938 jcspectra[ip, :, ind_hei] = jcspectra[ip,
1914 :, ind_hei] - jcspc_interf
1939 :, ind_hei] - jcspc_interf
1915
1940
1916 ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
1941 ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
1917 maxid = ListAux.index(max(ListAux))
1942 maxid = ListAux.index(max(ListAux))
1918
1943
1919 ind = numpy.array([-2, -1, 1, 2])
1944 ind = numpy.array([-2, -1, 1, 2])
1920 xx = numpy.zeros([4, 4])
1945 xx = numpy.zeros([4, 4])
1921
1946
1922 for id1 in range(4):
1947 for id1 in range(4):
1923 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1948 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1924
1949
1925 xx_inv = numpy.linalg.inv(xx)
1950 xx_inv = numpy.linalg.inv(xx)
1926 xx = xx_inv[:, 0]
1951 xx = xx_inv[:, 0]
1927
1952
1928 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1953 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1929 yy = jcspectra[ip, mask_prof[ind], :]
1954 yy = jcspectra[ip, mask_prof[ind], :]
1930 jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
1955 jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
1931
1956
1932 # Guardar Resultados
1957 # Guardar Resultados
1933 self.dataOut.data_spc = jspectra
1958 self.dataOut.data_spc = jspectra
1934 self.dataOut.data_cspc = jcspectra
1959 self.dataOut.data_cspc = jcspectra
1935
1960
1936 return 1
1961 return 1
1937
1962
1938 def run(self, dataOut, interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None, mode=1, minHei=None, maxHei=None):
1963 def run(self, dataOut, interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None, mode=1, minHei=None, maxHei=None):
1939
1964
1940 self.dataOut = dataOut
1965 self.dataOut = dataOut
1941
1966
1942 if mode == 1:
1967 if mode == 1:
1943 self.removeInterference(interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None)
1968 self.removeInterference(interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None)
1944 elif mode == 2:
1969 elif mode == 2:
1945 self.removeInterference2()
1970 self.removeInterference2()
1946 elif mode == 3:
1971 elif mode == 3:
1947 self.removeInterference3(min_hei=minHei, max_hei=maxHei)
1972 self.removeInterference3(min_hei=minHei, max_hei=maxHei)
1948 return self.dataOut
1973 return self.dataOut
1949
1974
1950
1975
1951 class IncohInt(Operation):
1976 class IncohInt(Operation):
1952
1977
1953 __profIndex = 0
1978 __profIndex = 0
1954 __withOverapping = False
1979 __withOverapping = False
1955
1980
1956 __byTime = False
1981 __byTime = False
1957 __initime = None
1982 __initime = None
1958 __lastdatatime = None
1983 __lastdatatime = None
1959 __integrationtime = None
1984 __integrationtime = None
1960
1985
1961 __buffer_spc = None
1986 __buffer_spc = None
1962 __buffer_cspc = None
1987 __buffer_cspc = None
1963 __buffer_dc = None
1988 __buffer_dc = None
1964
1989
1965 __dataReady = False
1990 __dataReady = False
1966
1991
1967 __timeInterval = None
1992 __timeInterval = None
1968 incohInt = 0
1993 incohInt = 0
1969 nOutliers = 0
1994 nOutliers = 0
1970 n = None
1995 n = None
1971
1996
1972 def __init__(self):
1997 def __init__(self):
1973
1998
1974 Operation.__init__(self)
1999 Operation.__init__(self)
1975
2000
1976 def setup(self, n=None, timeInterval=None, overlapping=False):
2001 def setup(self, n=None, timeInterval=None, overlapping=False):
1977 """
2002 """
1978 Set the parameters of the integration class.
2003 Set the parameters of the integration class.
1979
2004
1980 Inputs:
2005 Inputs:
1981
2006
1982 n : Number of coherent integrations
2007 n : Number of coherent integrations
1983 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
2008 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1984 overlapping :
2009 overlapping :
1985
2010
1986 """
2011 """
1987
2012
1988 self.__initime = None
2013 self.__initime = None
1989 self.__lastdatatime = 0
2014 self.__lastdatatime = 0
1990
2015
1991 self.__buffer_spc = 0
2016 self.__buffer_spc = 0
1992 self.__buffer_cspc = 0
2017 self.__buffer_cspc = 0
1993 self.__buffer_dc = 0
2018 self.__buffer_dc = 0
1994
2019
1995 self.__profIndex = 0
2020 self.__profIndex = 0
1996 self.__dataReady = False
2021 self.__dataReady = False
1997 self.__byTime = False
2022 self.__byTime = False
1998 self.incohInt = 0
2023 self.incohInt = 0
1999 self.nOutliers = 0
2024 self.nOutliers = 0
2000 if n is None and timeInterval is None:
2025 if n is None and timeInterval is None:
2001 raise ValueError("n or timeInterval should be specified ...")
2026 raise ValueError("n or timeInterval should be specified ...")
2002
2027
2003 if n is not None:
2028 if n is not None:
2004 self.n = int(n)
2029 self.n = int(n)
2005 else:
2030 else:
2006
2031
2007 self.__integrationtime = int(timeInterval)
2032 self.__integrationtime = int(timeInterval)
2008 self.n = None
2033 self.n = None
2009 self.__byTime = True
2034 self.__byTime = True
2010
2035
2036
2011 def putData(self, data_spc, data_cspc, data_dc):
2037 def putData(self, data_spc, data_cspc, data_dc):
2012 """
2038 """
2013 Add a profile to the __buffer_spc and increase in one the __profileIndex
2039 Add a profile to the __buffer_spc and increase in one the __profileIndex
2014
2040
2015 """
2041 """
2016 if data_spc.all() == numpy.nan :
2042 if data_spc.all() == numpy.nan :
2017 print("nan ")
2043 print("nan ")
2018 return
2044 return
2019 self.__buffer_spc += data_spc
2045 self.__buffer_spc += data_spc
2020
2046
2021 if data_cspc is None:
2047 if data_cspc is None:
2022 self.__buffer_cspc = None
2048 self.__buffer_cspc = None
2023 else:
2049 else:
2024 self.__buffer_cspc += data_cspc
2050 self.__buffer_cspc += data_cspc
2025
2051
2026 if data_dc is None:
2052 if data_dc is None:
2027 self.__buffer_dc = None
2053 self.__buffer_dc = None
2028 else:
2054 else:
2029 self.__buffer_dc += data_dc
2055 self.__buffer_dc += data_dc
2030
2056
2031 self.__profIndex += 1
2057 self.__profIndex += 1
2032
2058
2033 return
2059 return
2034
2060
2035 def pushData(self):
2061 def pushData(self):
2036 """
2062 """
2037 Return the sum of the last profiles and the profiles used in the sum.
2063 Return the sum of the last profiles and the profiles used in the sum.
2038
2064
2039 Affected:
2065 Affected:
2040
2066
2041 self.__profileIndex
2067 self.__profileIndex
2042
2068
2043 """
2069 """
2044
2070
2045 data_spc = self.__buffer_spc
2071 data_spc = self.__buffer_spc
2046 data_cspc = self.__buffer_cspc
2072 data_cspc = self.__buffer_cspc
2047 data_dc = self.__buffer_dc
2073 data_dc = self.__buffer_dc
2048 n = self.__profIndex
2074 n = self.__profIndex
2049
2075
2050 self.__buffer_spc = 0
2076 self.__buffer_spc = 0
2051 self.__buffer_cspc = 0
2077 self.__buffer_cspc = 0
2052 self.__buffer_dc = 0
2078 self.__buffer_dc = 0
2053
2079
2054
2080
2055 return data_spc, data_cspc, data_dc, n
2081 return data_spc, data_cspc, data_dc, n
2056
2082
2057 def byProfiles(self, *args):
2083 def byProfiles(self, *args):
2058
2084
2059 self.__dataReady = False
2085 self.__dataReady = False
2060 avgdata_spc = None
2086 avgdata_spc = None
2061 avgdata_cspc = None
2087 avgdata_cspc = None
2062 avgdata_dc = None
2088 avgdata_dc = None
2063
2089
2064 self.putData(*args)
2090 self.putData(*args)
2065
2091
2066 if self.__profIndex == self.n:
2092 if self.__profIndex == self.n:
2067
2093
2068 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
2094 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
2069 self.n = n
2095 self.n = n
2070 self.__dataReady = True
2096 self.__dataReady = True
2071
2097
2072 return avgdata_spc, avgdata_cspc, avgdata_dc
2098 return avgdata_spc, avgdata_cspc, avgdata_dc
2073
2099
2074 def byTime(self, datatime, *args):
2100 def byTime(self, datatime, *args):
2075
2101
2076 self.__dataReady = False
2102 self.__dataReady = False
2077 avgdata_spc = None
2103 avgdata_spc = None
2078 avgdata_cspc = None
2104 avgdata_cspc = None
2079 avgdata_dc = None
2105 avgdata_dc = None
2080
2106
2081 self.putData(*args)
2107 self.putData(*args)
2082
2108
2083 if (datatime - self.__initime) >= self.__integrationtime:
2109 if (datatime - self.__initime) >= self.__integrationtime:
2084 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
2110 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
2085 self.n = n
2111 self.n = n
2086 self.__dataReady = True
2112 self.__dataReady = True
2087
2113
2088 return avgdata_spc, avgdata_cspc, avgdata_dc
2114 return avgdata_spc, avgdata_cspc, avgdata_dc
2089
2115
2090 def integrate(self, datatime, *args):
2116 def integrate(self, datatime, *args):
2091
2117
2092 if self.__profIndex == 0:
2118 if self.__profIndex == 0:
2093 self.__initime = datatime
2119 self.__initime = datatime
2094
2120
2095 if self.__byTime:
2121 if self.__byTime:
2096 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
2122 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
2097 datatime, *args)
2123 datatime, *args)
2098 else:
2124 else:
2099 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
2125 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
2100
2126
2101 if not self.__dataReady:
2127 if not self.__dataReady:
2102 return None, None, None, None
2128 return None, None, None, None
2103
2129
2104 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
2130 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
2105
2131
2106 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
2132 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
2107 if n == 1:
2133 if n == 1:
2108 return dataOut
2134 return dataOut
2109
2135
2110 if dataOut.flagNoData == True:
2136 if dataOut.flagNoData == True:
2111 return dataOut
2137 return dataOut
2112
2138
2113 dataOut.flagNoData = True
2139 dataOut.flagNoData = True
2114
2140 dataOut.processingHeaderObj.timeIncohInt = timeInterval
2115 if not self.isConfig:
2141 if not self.isConfig:
2116 self.setup(n, timeInterval, overlapping)
2142 self.setup(n, timeInterval, overlapping)
2117 self.isConfig = True
2143 self.isConfig = True
2118
2144
2145
2119 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
2146 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
2120 dataOut.data_spc,
2147 dataOut.data_spc,
2121 dataOut.data_cspc,
2148 dataOut.data_cspc,
2122 dataOut.data_dc)
2149 dataOut.data_dc)
2123
2150
2124 self.incohInt += dataOut.nIncohInt
2151 self.incohInt += dataOut.nIncohInt
2125
2152
2126 if isinstance(dataOut.data_outlier,numpy.ndarray) or isinstance(dataOut.data_outlier,int) or isinstance(dataOut.data_outlier, float):
2153 if isinstance(dataOut.data_outlier,numpy.ndarray) or isinstance(dataOut.data_outlier,int) or isinstance(dataOut.data_outlier, float):
2127 self.nOutliers += dataOut.data_outlier
2154 self.nOutliers += dataOut.data_outlier
2128
2155
2129 if self.__dataReady:
2156 if self.__dataReady:
2130 #print("prof: ",dataOut.max_nIncohInt,self.__profIndex)
2157 #print("prof: ",dataOut.max_nIncohInt,self.__profIndex)
2131 dataOut.data_spc = avgdata_spc
2158 dataOut.data_spc = avgdata_spc
2132 dataOut.data_cspc = avgdata_cspc
2159 dataOut.data_cspc = avgdata_cspc
2133 dataOut.data_dc = avgdata_dc
2160 dataOut.data_dc = avgdata_dc
2134 dataOut.nIncohInt = self.incohInt
2161 dataOut.nIncohInt = self.incohInt
2135 dataOut.data_outlier = self.nOutliers
2162 dataOut.data_outlier = self.nOutliers
2136 dataOut.utctime = avgdatatime
2163 dataOut.utctime = avgdatatime
2137 dataOut.flagNoData = False
2164 dataOut.flagNoData = False
2138 self.incohInt = 0
2165 self.incohInt = 0
2139 self.nOutliers = 0
2166 self.nOutliers = 0
2140 self.__profIndex = 0
2167 self.__profIndex = 0
2141 #print("IncohInt Done")
2168 #print("IncohInt Done")
2142 return dataOut
2169 return dataOut
2143
2170
2144 class dopplerFlip(Operation):
2171 class dopplerFlip(Operation):
2145
2172
2146 def run(self, dataOut):
2173 def run(self, dataOut):
2147 # arreglo 1: (num_chan, num_profiles, num_heights)
2174 # arreglo 1: (num_chan, num_profiles, num_heights)
2148 self.dataOut = dataOut
2175 self.dataOut = dataOut
2149 # JULIA-oblicua, indice 2
2176 # JULIA-oblicua, indice 2
2150 # arreglo 2: (num_profiles, num_heights)
2177 # arreglo 2: (num_profiles, num_heights)
2151 jspectra = self.dataOut.data_spc[2]
2178 jspectra = self.dataOut.data_spc[2]
2152 jspectra_tmp = numpy.zeros(jspectra.shape)
2179 jspectra_tmp = numpy.zeros(jspectra.shape)
2153 num_profiles = jspectra.shape[0]
2180 num_profiles = jspectra.shape[0]
2154 freq_dc = int(num_profiles / 2)
2181 freq_dc = int(num_profiles / 2)
2155 # Flip con for
2182 # Flip con for
2156 for j in range(num_profiles):
2183 for j in range(num_profiles):
2157 jspectra_tmp[num_profiles-j-1]= jspectra[j]
2184 jspectra_tmp[num_profiles-j-1]= jspectra[j]
2158 # Intercambio perfil de DC con perfil inmediato anterior
2185 # Intercambio perfil de DC con perfil inmediato anterior
2159 jspectra_tmp[freq_dc-1]= jspectra[freq_dc-1]
2186 jspectra_tmp[freq_dc-1]= jspectra[freq_dc-1]
2160 jspectra_tmp[freq_dc]= jspectra[freq_dc]
2187 jspectra_tmp[freq_dc]= jspectra[freq_dc]
2161 # canal modificado es re-escrito en el arreglo de canales
2188 # canal modificado es re-escrito en el arreglo de canales
2162 self.dataOut.data_spc[2] = jspectra_tmp
2189 self.dataOut.data_spc[2] = jspectra_tmp
2163
2190
2164 return self.dataOut
2191 return self.dataOut
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
General Comments 0
You need to be logged in to leave comments. Login now