##// END OF EJS Templates
schain-jc
RSS
Multiprocessing for writing Units(Spectral, Voltage and Parameters)
George Yong -
r1179:6414333e2ace
parent child
Show More
Side by Side Unified
Context file:
r1179:6414333e2ace Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -1,1234 +1,1251
1 '''
1 '''
2
2
3 $Author: murco $
3 $Author: murco $
4 $Id: JROData.py 173 2012-11-20 15:06:21Z murco $
4 $Id: JROData.py 173 2012-11-20 15:06:21Z murco $
5 '''
5 '''
6
6
7 import copy
7 import copy
8 import numpy
8 import numpy
9 import datetime
9 import datetime
10
10
11 from .jroheaderIO import SystemHeader, RadarControllerHeader
11 from .jroheaderIO import SystemHeader, RadarControllerHeader
12
12
13
13
14 def getNumpyDtype(dataTypeCode):
14 def getNumpyDtype(dataTypeCode):
15
15
16 if dataTypeCode == 0:
16 if dataTypeCode == 0:
17 numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
17 numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
18 elif dataTypeCode == 1:
18 elif dataTypeCode == 1:
19 numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
19 numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
20 elif dataTypeCode == 2:
20 elif dataTypeCode == 2:
21 numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
21 numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
22 elif dataTypeCode == 3:
22 elif dataTypeCode == 3:
23 numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
23 numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
24 elif dataTypeCode == 4:
24 elif dataTypeCode == 4:
25 numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
25 numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
26 elif dataTypeCode == 5:
26 elif dataTypeCode == 5:
27 numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
27 numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
28 else:
28 else:
29 raise ValueError('dataTypeCode was not defined')
29 raise ValueError('dataTypeCode was not defined')
30
30
31 return numpyDtype
31 return numpyDtype
32
32
33
33
34 def getDataTypeCode(numpyDtype):
34 def getDataTypeCode(numpyDtype):
35
35
36 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
36 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
37 datatype = 0
37 datatype = 0
38 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
38 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
39 datatype = 1
39 datatype = 1
40 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
40 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
41 datatype = 2
41 datatype = 2
42 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
42 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
43 datatype = 3
43 datatype = 3
44 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
44 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
45 datatype = 4
45 datatype = 4
46 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
46 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
47 datatype = 5
47 datatype = 5
48 else:
48 else:
49 datatype = None
49 datatype = None
50
50
51 return datatype
51 return datatype
52
52
53
53
54 def hildebrand_sekhon(data, navg):
54 def hildebrand_sekhon(data, navg):
55 """
55 """
56 This method is for the objective determination of the noise level in Doppler spectra. This
56 This method is for the objective determination of the noise level in Doppler spectra. This
57 implementation technique is based on the fact that the standard deviation of the spectral
57 implementation technique is based on the fact that the standard deviation of the spectral
58 densities is equal to the mean spectral density for white Gaussian noise
58 densities is equal to the mean spectral density for white Gaussian noise
59
59
60 Inputs:
60 Inputs:
61 Data : heights
61 Data : heights
62 navg : numbers of averages
62 navg : numbers of averages
63
63
64 Return:
64 Return:
65 mean : noise's level
65 mean : noise's level
66 """
66 """
67
67
68 sorted_spectrum = numpy.sort(data, axis=None)
68 sortdata = numpy.sort(data, axis=None)
69 nnoise = len(sorted_spectrum) # default to all points in the spectrum as noise
69 lenOfData = len(sortdata)
70 for npts in range(1, len(sorted_spectrum)+1):
70 nums_min = lenOfData*0.2
71 partial = sorted_spectrum[:npts]
71
72 mean = partial.mean()
72 if nums_min <= 5:
73 var = partial.var()
73
74 if var * navg < mean**2.:
74 nums_min = 5
75 nnoise = npts
75
76 else:
76 sump = 0.
77 # partial spectrum no longer has characteristics of white noise
77 sumq = 0.
78 break
78
79 j = 0
80 cont = 1
81
82 while((cont==1)and(j<lenOfData)):
83
84 sump += sortdata[j]
85 sumq += sortdata[j]**2
86
87 if j > nums_min:
88 rtest = float(j)/(j-1) + 1.0/navg
89 if ((sumq*j) > (rtest*sump**2)):
90 j = j - 1
91 sump = sump - sortdata[j]
92 sumq = sumq - sortdata[j]**2
93 cont = 0
94
95 j += 1
96
97 lnoise = sump /j
79
98
80 noise_spectrum = sorted_spectrum[:nnoise]
99 return lnoise
81 mean = noise_spectrum.mean()
82 return mean
83
100
84
101
85 class Beam:
102 class Beam:
86
103
87 def __init__(self):
104 def __init__(self):
88 self.codeList = []
105 self.codeList = []
89 self.azimuthList = []
106 self.azimuthList = []
90 self.zenithList = []
107 self.zenithList = []
91
108
92
109
93 class GenericData(object):
110 class GenericData(object):
94
111
95 flagNoData = True
112 flagNoData = True
96
113
97 def copy(self, inputObj=None):
114 def copy(self, inputObj=None):
98
115
99 if inputObj == None:
116 if inputObj == None:
100 return copy.deepcopy(self)
117 return copy.deepcopy(self)
101
118
102 for key in list(inputObj.__dict__.keys()):
119 for key in list(inputObj.__dict__.keys()):
103
120
104 attribute = inputObj.__dict__[key]
121 attribute = inputObj.__dict__[key]
105
122
106 # If this attribute is a tuple or list
123 # If this attribute is a tuple or list
107 if type(inputObj.__dict__[key]) in (tuple, list):
124 if type(inputObj.__dict__[key]) in (tuple, list):
108 self.__dict__[key] = attribute[:]
125 self.__dict__[key] = attribute[:]
109 continue
126 continue
110
127
111 # If this attribute is another object or instance
128 # If this attribute is another object or instance
112 if hasattr(attribute, '__dict__'):
129 if hasattr(attribute, '__dict__'):
113 self.__dict__[key] = attribute.copy()
130 self.__dict__[key] = attribute.copy()
114 continue
131 continue
115
132
116 self.__dict__[key] = inputObj.__dict__[key]
133 self.__dict__[key] = inputObj.__dict__[key]
117
134
118 def deepcopy(self):
135 def deepcopy(self):
119
136
120 return copy.deepcopy(self)
137 return copy.deepcopy(self)
121
138
122 def isEmpty(self):
139 def isEmpty(self):
123
140
124 return self.flagNoData
141 return self.flagNoData
125
142
126
143
127 class JROData(GenericData):
144 class JROData(GenericData):
128
145
129 # m_BasicHeader = BasicHeader()
146 # m_BasicHeader = BasicHeader()
130 # m_ProcessingHeader = ProcessingHeader()
147 # m_ProcessingHeader = ProcessingHeader()
131
148
132 systemHeaderObj = SystemHeader()
149 systemHeaderObj = SystemHeader()
133
150
134 radarControllerHeaderObj = RadarControllerHeader()
151 radarControllerHeaderObj = RadarControllerHeader()
135
152
136 # data = None
153 # data = None
137
154
138 type = None
155 type = None
139
156
140 datatype = None # dtype but in string
157 datatype = None # dtype but in string
141
158
142 # dtype = None
159 # dtype = None
143
160
144 # nChannels = None
161 # nChannels = None
145
162
146 # nHeights = None
163 # nHeights = None
147
164
148 nProfiles = None
165 nProfiles = None
149
166
150 heightList = None
167 heightList = None
151
168
152 channelList = None
169 channelList = None
153
170
154 flagDiscontinuousBlock = False
171 flagDiscontinuousBlock = False
155
172
156 useLocalTime = False
173 useLocalTime = False
157
174
158 utctime = None
175 utctime = None
159
176
160 timeZone = None
177 timeZone = None
161
178
162 dstFlag = None
179 dstFlag = None
163
180
164 errorCount = None
181 errorCount = None
165
182
166 blocksize = None
183 blocksize = None
167
184
168 # nCode = None
185 # nCode = None
169 #
186 #
170 # nBaud = None
187 # nBaud = None
171 #
188 #
172 # code = None
189 # code = None
173
190
174 flagDecodeData = False # asumo q la data no esta decodificada
191 flagDecodeData = False # asumo q la data no esta decodificada
175
192
176 flagDeflipData = False # asumo q la data no esta sin flip
193 flagDeflipData = False # asumo q la data no esta sin flip
177
194
178 flagShiftFFT = False
195 flagShiftFFT = False
179
196
180 # ippSeconds = None
197 # ippSeconds = None
181
198
182 # timeInterval = None
199 # timeInterval = None
183
200
184 nCohInt = None
201 nCohInt = None
185
202
186 # noise = None
203 # noise = None
187
204
188 windowOfFilter = 1
205 windowOfFilter = 1
189
206
190 # Speed of ligth
207 # Speed of ligth
191 C = 3e8
208 C = 3e8
192
209
193 frequency = 49.92e6
210 frequency = 49.92e6
194
211
195 realtime = False
212 realtime = False
196
213
197 beacon_heiIndexList = None
214 beacon_heiIndexList = None
198
215
199 last_block = None
216 last_block = None
200
217
201 blocknow = None
218 blocknow = None
202
219
203 azimuth = None
220 azimuth = None
204
221
205 zenith = None
222 zenith = None
206
223
207 beam = Beam()
224 beam = Beam()
208
225
209 profileIndex = None
226 profileIndex = None
210
227
211 error = (0, '')
228 error = (0, '')
212
229
213 def __str__(self):
230 def __str__(self):
214
231
215 return '{} - {}'.format(self.type, self.getDatatime())
232 return '{} - {}'.format(self.type, self.getDatatime())
216
233
217 def getNoise(self):
234 def getNoise(self):
218
235
219 raise NotImplementedError
236 raise NotImplementedError
220
237
221 def getNChannels(self):
238 def getNChannels(self):
222
239
223 return len(self.channelList)
240 return len(self.channelList)
224
241
225 def getChannelIndexList(self):
242 def getChannelIndexList(self):
226
243
227 return list(range(self.nChannels))
244 return list(range(self.nChannels))
228
245
229 def getNHeights(self):
246 def getNHeights(self):
230
247
231 return len(self.heightList)
248 return len(self.heightList)
232
249
233 def getHeiRange(self, extrapoints=0):
250 def getHeiRange(self, extrapoints=0):
234
251
235 heis = self.heightList
252 heis = self.heightList
236 # deltah = self.heightList[1] - self.heightList[0]
253 # deltah = self.heightList[1] - self.heightList[0]
237 #
254 #
238 # heis.append(self.heightList[-1])
255 # heis.append(self.heightList[-1])
239
256
240 return heis
257 return heis
241
258
242 def getDeltaH(self):
259 def getDeltaH(self):
243
260
244 delta = self.heightList[1] - self.heightList[0]
261 delta = self.heightList[1] - self.heightList[0]
245
262
246 return delta
263 return delta
247
264
248 def getltctime(self):
265 def getltctime(self):
249
266
250 if self.useLocalTime:
267 if self.useLocalTime:
251 return self.utctime - self.timeZone * 60
268 return self.utctime - self.timeZone * 60
252
269
253 return self.utctime
270 return self.utctime
254
271
255 def getDatatime(self):
272 def getDatatime(self):
256
273
257 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
274 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
258 return datatimeValue
275 return datatimeValue
259
276
260 def getTimeRange(self):
277 def getTimeRange(self):
261
278
262 datatime = []
279 datatime = []
263
280
264 datatime.append(self.ltctime)
281 datatime.append(self.ltctime)
265 datatime.append(self.ltctime + self.timeInterval + 1)
282 datatime.append(self.ltctime + self.timeInterval + 1)
266
283
267 datatime = numpy.array(datatime)
284 datatime = numpy.array(datatime)
268
285
269 return datatime
286 return datatime
270
287
271 def getFmaxTimeResponse(self):
288 def getFmaxTimeResponse(self):
272
289
273 period = (10**-6) * self.getDeltaH() / (0.15)
290 period = (10**-6) * self.getDeltaH() / (0.15)
274
291
275 PRF = 1. / (period * self.nCohInt)
292 PRF = 1. / (period * self.nCohInt)
276
293
277 fmax = PRF
294 fmax = PRF
278
295
279 return fmax
296 return fmax
280
297
281 def getFmax(self):
298 def getFmax(self):
282 PRF = 1. / (self.ippSeconds * self.nCohInt)
299 PRF = 1. / (self.ippSeconds * self.nCohInt)
283
300
284 fmax = PRF
301 fmax = PRF
285 return fmax
302 return fmax
286
303
287 def getVmax(self):
304 def getVmax(self):
288
305
289 _lambda = self.C / self.frequency
306 _lambda = self.C / self.frequency
290
307
291 vmax = self.getFmax() * _lambda / 2
308 vmax = self.getFmax() * _lambda / 2
292
309
293 return vmax
310 return vmax
294
311
295 def get_ippSeconds(self):
312 def get_ippSeconds(self):
296 '''
313 '''
297 '''
314 '''
298 return self.radarControllerHeaderObj.ippSeconds
315 return self.radarControllerHeaderObj.ippSeconds
299
316
300 def set_ippSeconds(self, ippSeconds):
317 def set_ippSeconds(self, ippSeconds):
301 '''
318 '''
302 '''
319 '''
303
320
304 self.radarControllerHeaderObj.ippSeconds = ippSeconds
321 self.radarControllerHeaderObj.ippSeconds = ippSeconds
305
322
306 return
323 return
307
324
308 def get_dtype(self):
325 def get_dtype(self):
309 '''
326 '''
310 '''
327 '''
311 return getNumpyDtype(self.datatype)
328 return getNumpyDtype(self.datatype)
312
329
313 def set_dtype(self, numpyDtype):
330 def set_dtype(self, numpyDtype):
314 '''
331 '''
315 '''
332 '''
316
333
317 self.datatype = getDataTypeCode(numpyDtype)
334 self.datatype = getDataTypeCode(numpyDtype)
318
335
319 def get_code(self):
336 def get_code(self):
320 '''
337 '''
321 '''
338 '''
322 return self.radarControllerHeaderObj.code
339 return self.radarControllerHeaderObj.code
323
340
324 def set_code(self, code):
341 def set_code(self, code):
325 '''
342 '''
326 '''
343 '''
327 self.radarControllerHeaderObj.code = code
344 self.radarControllerHeaderObj.code = code
328
345
329 return
346 return
330
347
331 def get_ncode(self):
348 def get_ncode(self):
332 '''
349 '''
333 '''
350 '''
334 return self.radarControllerHeaderObj.nCode
351 return self.radarControllerHeaderObj.nCode
335
352
336 def set_ncode(self, nCode):
353 def set_ncode(self, nCode):
337 '''
354 '''
338 '''
355 '''
339 self.radarControllerHeaderObj.nCode = nCode
356 self.radarControllerHeaderObj.nCode = nCode
340
357
341 return
358 return
342
359
343 def get_nbaud(self):
360 def get_nbaud(self):
344 '''
361 '''
345 '''
362 '''
346 return self.radarControllerHeaderObj.nBaud
363 return self.radarControllerHeaderObj.nBaud
347
364
348 def set_nbaud(self, nBaud):
365 def set_nbaud(self, nBaud):
349 '''
366 '''
350 '''
367 '''
351 self.radarControllerHeaderObj.nBaud = nBaud
368 self.radarControllerHeaderObj.nBaud = nBaud
352
369
353 return
370 return
354
371
355 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
372 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
356 channelIndexList = property(
373 channelIndexList = property(
357 getChannelIndexList, "I'm the 'channelIndexList' property.")
374 getChannelIndexList, "I'm the 'channelIndexList' property.")
358 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
375 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
359 #noise = property(getNoise, "I'm the 'nHeights' property.")
376 #noise = property(getNoise, "I'm the 'nHeights' property.")
360 datatime = property(getDatatime, "I'm the 'datatime' property")
377 datatime = property(getDatatime, "I'm the 'datatime' property")
361 ltctime = property(getltctime, "I'm the 'ltctime' property")
378 ltctime = property(getltctime, "I'm the 'ltctime' property")
362 ippSeconds = property(get_ippSeconds, set_ippSeconds)
379 ippSeconds = property(get_ippSeconds, set_ippSeconds)
363 dtype = property(get_dtype, set_dtype)
380 dtype = property(get_dtype, set_dtype)
364 # timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
381 # timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
365 code = property(get_code, set_code)
382 code = property(get_code, set_code)
366 nCode = property(get_ncode, set_ncode)
383 nCode = property(get_ncode, set_ncode)
367 nBaud = property(get_nbaud, set_nbaud)
384 nBaud = property(get_nbaud, set_nbaud)
368
385
369
386
370 class Voltage(JROData):
387 class Voltage(JROData):
371
388
372 # data es un numpy array de 2 dmensiones (canales, alturas)
389 # data es un numpy array de 2 dmensiones (canales, alturas)
373 data = None
390 data = None
374
391
375 def __init__(self):
392 def __init__(self):
376 '''
393 '''
377 Constructor
394 Constructor
378 '''
395 '''
379
396
380 self.useLocalTime = True
397 self.useLocalTime = True
381
398
382 self.radarControllerHeaderObj = RadarControllerHeader()
399 self.radarControllerHeaderObj = RadarControllerHeader()
383
400
384 self.systemHeaderObj = SystemHeader()
401 self.systemHeaderObj = SystemHeader()
385
402
386 self.type = "Voltage"
403 self.type = "Voltage"
387
404
388 self.data = None
405 self.data = None
389
406
390 # self.dtype = None
407 # self.dtype = None
391
408
392 # self.nChannels = 0
409 # self.nChannels = 0
393
410
394 # self.nHeights = 0
411 # self.nHeights = 0
395
412
396 self.nProfiles = None
413 self.nProfiles = None
397
414
398 self.heightList = None
415 self.heightList = None
399
416
400 self.channelList = None
417 self.channelList = None
401
418
402 # self.channelIndexList = None
419 # self.channelIndexList = None
403
420
404 self.flagNoData = True
421 self.flagNoData = True
405
422
406 self.flagDiscontinuousBlock = False
423 self.flagDiscontinuousBlock = False
407
424
408 self.utctime = None
425 self.utctime = None
409
426
410 self.timeZone = None
427 self.timeZone = None
411
428
412 self.dstFlag = None
429 self.dstFlag = None
413
430
414 self.errorCount = None
431 self.errorCount = None
415
432
416 self.nCohInt = None
433 self.nCohInt = None
417
434
418 self.blocksize = None
435 self.blocksize = None
419
436
420 self.flagDecodeData = False # asumo q la data no esta decodificada
437 self.flagDecodeData = False # asumo q la data no esta decodificada
421
438
422 self.flagDeflipData = False # asumo q la data no esta sin flip
439 self.flagDeflipData = False # asumo q la data no esta sin flip
423
440
424 self.flagShiftFFT = False
441 self.flagShiftFFT = False
425
442
426 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
443 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
427
444
428 self.profileIndex = 0
445 self.profileIndex = 0
429
446
430 def getNoisebyHildebrand(self, channel=None):
447 def getNoisebyHildebrand(self, channel=None):
431 """
448 """
432 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
449 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
433
450
434 Return:
451 Return:
435 noiselevel
452 noiselevel
436 """
453 """
437
454
438 if channel != None:
455 if channel != None:
439 data = self.data[channel]
456 data = self.data[channel]
440 nChannels = 1
457 nChannels = 1
441 else:
458 else:
442 data = self.data
459 data = self.data
443 nChannels = self.nChannels
460 nChannels = self.nChannels
444
461
445 noise = numpy.zeros(nChannels)
462 noise = numpy.zeros(nChannels)
446 power = data * numpy.conjugate(data)
463 power = data * numpy.conjugate(data)
447
464
448 for thisChannel in range(nChannels):
465 for thisChannel in range(nChannels):
449 if nChannels == 1:
466 if nChannels == 1:
450 daux = power[:].real
467 daux = power[:].real
451 else:
468 else:
452 daux = power[thisChannel, :].real
469 daux = power[thisChannel, :].real
453 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
470 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
454
471
455 return noise
472 return noise
456
473
457 def getNoise(self, type=1, channel=None):
474 def getNoise(self, type=1, channel=None):
458
475
459 if type == 1:
476 if type == 1:
460 noise = self.getNoisebyHildebrand(channel)
477 noise = self.getNoisebyHildebrand(channel)
461
478
462 return noise
479 return noise
463
480
464 def getPower(self, channel=None):
481 def getPower(self, channel=None):
465
482
466 if channel != None:
483 if channel != None:
467 data = self.data[channel]
484 data = self.data[channel]
468 else:
485 else:
469 data = self.data
486 data = self.data
470
487
471 power = data * numpy.conjugate(data)
488 power = data * numpy.conjugate(data)
472 powerdB = 10 * numpy.log10(power.real)
489 powerdB = 10 * numpy.log10(power.real)
473 powerdB = numpy.squeeze(powerdB)
490 powerdB = numpy.squeeze(powerdB)
474
491
475 return powerdB
492 return powerdB
476
493
477 def getTimeInterval(self):
494 def getTimeInterval(self):
478
495
479 timeInterval = self.ippSeconds * self.nCohInt
496 timeInterval = self.ippSeconds * self.nCohInt
480
497
481 return timeInterval
498 return timeInterval
482
499
483 noise = property(getNoise, "I'm the 'nHeights' property.")
500 noise = property(getNoise, "I'm the 'nHeights' property.")
484 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
501 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
485
502
486
503
487 class Spectra(JROData):
504 class Spectra(JROData):
488
505
489 # data spc es un numpy array de 2 dmensiones (canales, perfiles, alturas)
506 # data spc es un numpy array de 2 dmensiones (canales, perfiles, alturas)
490 data_spc = None
507 data_spc = None
491
508
492 # data cspc es un numpy array de 2 dmensiones (canales, pares, alturas)
509 # data cspc es un numpy array de 2 dmensiones (canales, pares, alturas)
493 data_cspc = None
510 data_cspc = None
494
511
495 # data dc es un numpy array de 2 dmensiones (canales, alturas)
512 # data dc es un numpy array de 2 dmensiones (canales, alturas)
496 data_dc = None
513 data_dc = None
497
514
498 # data power
515 # data power
499 data_pwr = None
516 data_pwr = None
500
517
501 nFFTPoints = None
518 nFFTPoints = None
502
519
503 # nPairs = None
520 # nPairs = None
504
521
505 pairsList = None
522 pairsList = None
506
523
507 nIncohInt = None
524 nIncohInt = None
508
525
509 wavelength = None # Necesario para cacular el rango de velocidad desde la frecuencia
526 wavelength = None # Necesario para cacular el rango de velocidad desde la frecuencia
510
527
511 nCohInt = None # se requiere para determinar el valor de timeInterval
528 nCohInt = None # se requiere para determinar el valor de timeInterval
512
529
513 ippFactor = None
530 ippFactor = None
514
531
515 profileIndex = 0
532 profileIndex = 0
516
533
517 plotting = "spectra"
534 plotting = "spectra"
518
535
519 def __init__(self):
536 def __init__(self):
520 '''
537 '''
521 Constructor
538 Constructor
522 '''
539 '''
523
540
524 self.useLocalTime = True
541 self.useLocalTime = True
525
542
526 self.radarControllerHeaderObj = RadarControllerHeader()
543 self.radarControllerHeaderObj = RadarControllerHeader()
527
544
528 self.systemHeaderObj = SystemHeader()
545 self.systemHeaderObj = SystemHeader()
529
546
530 self.type = "Spectra"
547 self.type = "Spectra"
531
548
532 # self.data = None
549 # self.data = None
533
550
534 # self.dtype = None
551 # self.dtype = None
535
552
536 # self.nChannels = 0
553 # self.nChannels = 0
537
554
538 # self.nHeights = 0
555 # self.nHeights = 0
539
556
540 self.nProfiles = None
557 self.nProfiles = None
541
558
542 self.heightList = None
559 self.heightList = None
543
560
544 self.channelList = None
561 self.channelList = None
545
562
546 # self.channelIndexList = None
563 # self.channelIndexList = None
547
564
548 self.pairsList = None
565 self.pairsList = None
549
566
550 self.flagNoData = True
567 self.flagNoData = True
551
568
552 self.flagDiscontinuousBlock = False
569 self.flagDiscontinuousBlock = False
553
570
554 self.utctime = None
571 self.utctime = None
555
572
556 self.nCohInt = None
573 self.nCohInt = None
557
574
558 self.nIncohInt = None
575 self.nIncohInt = None
559
576
560 self.blocksize = None
577 self.blocksize = None
561
578
562 self.nFFTPoints = None
579 self.nFFTPoints = None
563
580
564 self.wavelength = None
581 self.wavelength = None
565
582
566 self.flagDecodeData = False # asumo q la data no esta decodificada
583 self.flagDecodeData = False # asumo q la data no esta decodificada
567
584
568 self.flagDeflipData = False # asumo q la data no esta sin flip
585 self.flagDeflipData = False # asumo q la data no esta sin flip
569
586
570 self.flagShiftFFT = False
587 self.flagShiftFFT = False
571
588
572 self.ippFactor = 1
589 self.ippFactor = 1
573
590
574 #self.noise = None
591 #self.noise = None
575
592
576 self.beacon_heiIndexList = []
593 self.beacon_heiIndexList = []
577
594
578 self.noise_estimation = None
595 self.noise_estimation = None
579
596
580 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
597 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
581 """
598 """
582 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
599 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
583
600
584 Return:
601 Return:
585 noiselevel
602 noiselevel
586 """
603 """
587
604
588 noise = numpy.zeros(self.nChannels)
605 noise = numpy.zeros(self.nChannels)
589
606
590 for channel in range(self.nChannels):
607 for channel in range(self.nChannels):
591 daux = self.data_spc[channel,
608 daux = self.data_spc[channel,
592 xmin_index:xmax_index, ymin_index:ymax_index]
609 xmin_index:xmax_index, ymin_index:ymax_index]
593 noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
610 noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
594
611
595 return noise
612 return noise
596
613
597 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
614 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
598
615
599 if self.noise_estimation is not None:
616 if self.noise_estimation is not None:
600 # this was estimated by getNoise Operation defined in jroproc_spectra.py
617 # this was estimated by getNoise Operation defined in jroproc_spectra.py
601 return self.noise_estimation
618 return self.noise_estimation
602 else:
619 else:
603 noise = self.getNoisebyHildebrand(
620 noise = self.getNoisebyHildebrand(
604 xmin_index, xmax_index, ymin_index, ymax_index)
621 xmin_index, xmax_index, ymin_index, ymax_index)
605 return noise
622 return noise
606
623
607 def getFreqRangeTimeResponse(self, extrapoints=0):
624 def getFreqRangeTimeResponse(self, extrapoints=0):
608
625
609 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
626 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
610 freqrange = deltafreq * \
627 freqrange = deltafreq * \
611 (numpy.arange(self.nFFTPoints + extrapoints) -
628 (numpy.arange(self.nFFTPoints + extrapoints) -
612 self.nFFTPoints / 2.) - deltafreq / 2
629 self.nFFTPoints / 2.) - deltafreq / 2
613
630
614 return freqrange
631 return freqrange
615
632
616 def getAcfRange(self, extrapoints=0):
633 def getAcfRange(self, extrapoints=0):
617
634
618 deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
635 deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
619 freqrange = deltafreq * \
636 freqrange = deltafreq * \
620 (numpy.arange(self.nFFTPoints + extrapoints) -
637 (numpy.arange(self.nFFTPoints + extrapoints) -
621 self.nFFTPoints / 2.) - deltafreq / 2
638 self.nFFTPoints / 2.) - deltafreq / 2
622
639
623 return freqrange
640 return freqrange
624
641
625 def getFreqRange(self, extrapoints=0):
642 def getFreqRange(self, extrapoints=0):
626
643
627 deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
644 deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
628 freqrange = deltafreq * \
645 freqrange = deltafreq * \
629 (numpy.arange(self.nFFTPoints + extrapoints) -
646 (numpy.arange(self.nFFTPoints + extrapoints) -
630 self.nFFTPoints / 2.) - deltafreq / 2
647 self.nFFTPoints / 2.) - deltafreq / 2
631
648
632 return freqrange
649 return freqrange
633
650
634 def getVelRange(self, extrapoints=0):
651 def getVelRange(self, extrapoints=0):
635
652
636 deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
653 deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
637 velrange = deltav * (numpy.arange(self.nFFTPoints +
654 velrange = deltav * (numpy.arange(self.nFFTPoints +
638 extrapoints) - self.nFFTPoints / 2.) # - deltav/2
655 extrapoints) - self.nFFTPoints / 2.) # - deltav/2
639
656
640 return velrange
657 return velrange
641
658
642 def getNPairs(self):
659 def getNPairs(self):
643
660
644 return len(self.pairsList)
661 return len(self.pairsList)
645
662
646 def getPairsIndexList(self):
663 def getPairsIndexList(self):
647
664
648 return list(range(self.nPairs))
665 return list(range(self.nPairs))
649
666
650 def getNormFactor(self):
667 def getNormFactor(self):
651
668
652 pwcode = 1
669 pwcode = 1
653
670
654 if self.flagDecodeData:
671 if self.flagDecodeData:
655 pwcode = numpy.sum(self.code[0]**2)
672 pwcode = numpy.sum(self.code[0]**2)
656 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
673 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
657 normFactor = self.nProfiles * self.nIncohInt * \
674 normFactor = self.nProfiles * self.nIncohInt * \
658 self.nCohInt * pwcode * self.windowOfFilter
675 self.nCohInt * pwcode * self.windowOfFilter
659
676
660 return normFactor
677 return normFactor
661
678
662 def getFlagCspc(self):
679 def getFlagCspc(self):
663
680
664 if self.data_cspc is None:
681 if self.data_cspc is None:
665 return True
682 return True
666
683
667 return False
684 return False
668
685
669 def getFlagDc(self):
686 def getFlagDc(self):
670
687
671 if self.data_dc is None:
688 if self.data_dc is None:
672 return True
689 return True
673
690
674 return False
691 return False
675
692
676 def getTimeInterval(self):
693 def getTimeInterval(self):
677
694
678 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
695 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
679
696
680 return timeInterval
697 return timeInterval
681
698
682 def getPower(self):
699 def getPower(self):
683
700
684 factor = self.normFactor
701 factor = self.normFactor
685 z = self.data_spc / factor
702 z = self.data_spc / factor
686 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
703 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
687 avg = numpy.average(z, axis=1)
704 avg = numpy.average(z, axis=1)
688
705
689 return 10 * numpy.log10(avg)
706 return 10 * numpy.log10(avg)
690
707
691 def getCoherence(self, pairsList=None, phase=False):
708 def getCoherence(self, pairsList=None, phase=False):
692
709
693 z = []
710 z = []
694 if pairsList is None:
711 if pairsList is None:
695 pairsIndexList = self.pairsIndexList
712 pairsIndexList = self.pairsIndexList
696 else:
713 else:
697 pairsIndexList = []
714 pairsIndexList = []
698 for pair in pairsList:
715 for pair in pairsList:
699 if pair not in self.pairsList:
716 if pair not in self.pairsList:
700 raise ValueError("Pair %s is not in dataOut.pairsList" % (
717 raise ValueError("Pair %s is not in dataOut.pairsList" % (
701 pair))
718 pair))
702 pairsIndexList.append(self.pairsList.index(pair))
719 pairsIndexList.append(self.pairsList.index(pair))
703 for i in range(len(pairsIndexList)):
720 for i in range(len(pairsIndexList)):
704 pair = self.pairsList[pairsIndexList[i]]
721 pair = self.pairsList[pairsIndexList[i]]
705 ccf = numpy.average(
722 ccf = numpy.average(
706 self.data_cspc[pairsIndexList[i], :, :], axis=0)
723 self.data_cspc[pairsIndexList[i], :, :], axis=0)
707 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
724 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
708 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
725 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
709 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
726 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
710 if phase:
727 if phase:
711 data = numpy.arctan2(avgcoherenceComplex.imag,
728 data = numpy.arctan2(avgcoherenceComplex.imag,
712 avgcoherenceComplex.real) * 180 / numpy.pi
729 avgcoherenceComplex.real) * 180 / numpy.pi
713 else:
730 else:
714 data = numpy.abs(avgcoherenceComplex)
731 data = numpy.abs(avgcoherenceComplex)
715
732
716 z.append(data)
733 z.append(data)
717
734
718 return numpy.array(z)
735 return numpy.array(z)
719
736
720 def setValue(self, value):
737 def setValue(self, value):
721
738
722 print("This property should not be initialized")
739 print("This property should not be initialized")
723
740
724 return
741 return
725
742
726 nPairs = property(getNPairs, setValue, "I'm the 'nPairs' property.")
743 nPairs = property(getNPairs, setValue, "I'm the 'nPairs' property.")
727 pairsIndexList = property(
744 pairsIndexList = property(
728 getPairsIndexList, setValue, "I'm the 'pairsIndexList' property.")
745 getPairsIndexList, setValue, "I'm the 'pairsIndexList' property.")
729 normFactor = property(getNormFactor, setValue,
746 normFactor = property(getNormFactor, setValue,
730 "I'm the 'getNormFactor' property.")
747 "I'm the 'getNormFactor' property.")
731 flag_cspc = property(getFlagCspc, setValue)
748 flag_cspc = property(getFlagCspc, setValue)
732 flag_dc = property(getFlagDc, setValue)
749 flag_dc = property(getFlagDc, setValue)
733 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
750 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
734 timeInterval = property(getTimeInterval, setValue,
751 timeInterval = property(getTimeInterval, setValue,
735 "I'm the 'timeInterval' property")
752 "I'm the 'timeInterval' property")
736
753
737
754
738 class SpectraHeis(Spectra):
755 class SpectraHeis(Spectra):
739
756
740 data_spc = None
757 data_spc = None
741
758
742 data_cspc = None
759 data_cspc = None
743
760
744 data_dc = None
761 data_dc = None
745
762
746 nFFTPoints = None
763 nFFTPoints = None
747
764
748 # nPairs = None
765 # nPairs = None
749
766
750 pairsList = None
767 pairsList = None
751
768
752 nCohInt = None
769 nCohInt = None
753
770
754 nIncohInt = None
771 nIncohInt = None
755
772
756 def __init__(self):
773 def __init__(self):
757
774
758 self.radarControllerHeaderObj = RadarControllerHeader()
775 self.radarControllerHeaderObj = RadarControllerHeader()
759
776
760 self.systemHeaderObj = SystemHeader()
777 self.systemHeaderObj = SystemHeader()
761
778
762 self.type = "SpectraHeis"
779 self.type = "SpectraHeis"
763
780
764 # self.dtype = None
781 # self.dtype = None
765
782
766 # self.nChannels = 0
783 # self.nChannels = 0
767
784
768 # self.nHeights = 0
785 # self.nHeights = 0
769
786
770 self.nProfiles = None
787 self.nProfiles = None
771
788
772 self.heightList = None
789 self.heightList = None
773
790
774 self.channelList = None
791 self.channelList = None
775
792
776 # self.channelIndexList = None
793 # self.channelIndexList = None
777
794
778 self.flagNoData = True
795 self.flagNoData = True
779
796
780 self.flagDiscontinuousBlock = False
797 self.flagDiscontinuousBlock = False
781
798
782 # self.nPairs = 0
799 # self.nPairs = 0
783
800
784 self.utctime = None
801 self.utctime = None
785
802
786 self.blocksize = None
803 self.blocksize = None
787
804
788 self.profileIndex = 0
805 self.profileIndex = 0
789
806
790 self.nCohInt = 1
807 self.nCohInt = 1
791
808
792 self.nIncohInt = 1
809 self.nIncohInt = 1
793
810
794 def getNormFactor(self):
811 def getNormFactor(self):
795 pwcode = 1
812 pwcode = 1
796 if self.flagDecodeData:
813 if self.flagDecodeData:
797 pwcode = numpy.sum(self.code[0]**2)
814 pwcode = numpy.sum(self.code[0]**2)
798
815
799 normFactor = self.nIncohInt * self.nCohInt * pwcode
816 normFactor = self.nIncohInt * self.nCohInt * pwcode
800
817
801 return normFactor
818 return normFactor
802
819
803 def getTimeInterval(self):
820 def getTimeInterval(self):
804
821
805 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
822 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
806
823
807 return timeInterval
824 return timeInterval
808
825
809 normFactor = property(getNormFactor, "I'm the 'getNormFactor' property.")
826 normFactor = property(getNormFactor, "I'm the 'getNormFactor' property.")
810 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
827 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
811
828
812
829
813 class Fits(JROData):
830 class Fits(JROData):
814
831
815 heightList = None
832 heightList = None
816
833
817 channelList = None
834 channelList = None
818
835
819 flagNoData = True
836 flagNoData = True
820
837
821 flagDiscontinuousBlock = False
838 flagDiscontinuousBlock = False
822
839
823 useLocalTime = False
840 useLocalTime = False
824
841
825 utctime = None
842 utctime = None
826
843
827 timeZone = None
844 timeZone = None
828
845
829 # ippSeconds = None
846 # ippSeconds = None
830
847
831 # timeInterval = None
848 # timeInterval = None
832
849
833 nCohInt = None
850 nCohInt = None
834
851
835 nIncohInt = None
852 nIncohInt = None
836
853
837 noise = None
854 noise = None
838
855
839 windowOfFilter = 1
856 windowOfFilter = 1
840
857
841 # Speed of ligth
858 # Speed of ligth
842 C = 3e8
859 C = 3e8
843
860
844 frequency = 49.92e6
861 frequency = 49.92e6
845
862
846 realtime = False
863 realtime = False
847
864
848 def __init__(self):
865 def __init__(self):
849
866
850 self.type = "Fits"
867 self.type = "Fits"
851
868
852 self.nProfiles = None
869 self.nProfiles = None
853
870
854 self.heightList = None
871 self.heightList = None
855
872
856 self.channelList = None
873 self.channelList = None
857
874
858 # self.channelIndexList = None
875 # self.channelIndexList = None
859
876
860 self.flagNoData = True
877 self.flagNoData = True
861
878
862 self.utctime = None
879 self.utctime = None
863
880
864 self.nCohInt = 1
881 self.nCohInt = 1
865
882
866 self.nIncohInt = 1
883 self.nIncohInt = 1
867
884
868 self.useLocalTime = True
885 self.useLocalTime = True
869
886
870 self.profileIndex = 0
887 self.profileIndex = 0
871
888
872 # self.utctime = None
889 # self.utctime = None
873 # self.timeZone = None
890 # self.timeZone = None
874 # self.ltctime = None
891 # self.ltctime = None
875 # self.timeInterval = None
892 # self.timeInterval = None
876 # self.header = None
893 # self.header = None
877 # self.data_header = None
894 # self.data_header = None
878 # self.data = None
895 # self.data = None
879 # self.datatime = None
896 # self.datatime = None
880 # self.flagNoData = False
897 # self.flagNoData = False
881 # self.expName = ''
898 # self.expName = ''
882 # self.nChannels = None
899 # self.nChannels = None
883 # self.nSamples = None
900 # self.nSamples = None
884 # self.dataBlocksPerFile = None
901 # self.dataBlocksPerFile = None
885 # self.comments = ''
902 # self.comments = ''
886 #
903 #
887
904
888 def getltctime(self):
905 def getltctime(self):
889
906
890 if self.useLocalTime:
907 if self.useLocalTime:
891 return self.utctime - self.timeZone * 60
908 return self.utctime - self.timeZone * 60
892
909
893 return self.utctime
910 return self.utctime
894
911
895 def getDatatime(self):
912 def getDatatime(self):
896
913
897 datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
914 datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
898 return datatime
915 return datatime
899
916
900 def getTimeRange(self):
917 def getTimeRange(self):
901
918
902 datatime = []
919 datatime = []
903
920
904 datatime.append(self.ltctime)
921 datatime.append(self.ltctime)
905 datatime.append(self.ltctime + self.timeInterval)
922 datatime.append(self.ltctime + self.timeInterval)
906
923
907 datatime = numpy.array(datatime)
924 datatime = numpy.array(datatime)
908
925
909 return datatime
926 return datatime
910
927
911 def getHeiRange(self):
928 def getHeiRange(self):
912
929
913 heis = self.heightList
930 heis = self.heightList
914
931
915 return heis
932 return heis
916
933
917 def getNHeights(self):
934 def getNHeights(self):
918
935
919 return len(self.heightList)
936 return len(self.heightList)
920
937
921 def getNChannels(self):
938 def getNChannels(self):
922
939
923 return len(self.channelList)
940 return len(self.channelList)
924
941
925 def getChannelIndexList(self):
942 def getChannelIndexList(self):
926
943
927 return list(range(self.nChannels))
944 return list(range(self.nChannels))
928
945
929 def getNoise(self, type=1):
946 def getNoise(self, type=1):
930
947
931 #noise = numpy.zeros(self.nChannels)
948 #noise = numpy.zeros(self.nChannels)
932
949
933 if type == 1:
950 if type == 1:
934 noise = self.getNoisebyHildebrand()
951 noise = self.getNoisebyHildebrand()
935
952
936 if type == 2:
953 if type == 2:
937 noise = self.getNoisebySort()
954 noise = self.getNoisebySort()
938
955
939 if type == 3:
956 if type == 3:
940 noise = self.getNoisebyWindow()
957 noise = self.getNoisebyWindow()
941
958
942 return noise
959 return noise
943
960
944 def getTimeInterval(self):
961 def getTimeInterval(self):
945
962
946 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
963 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
947
964
948 return timeInterval
965 return timeInterval
949
966
950 datatime = property(getDatatime, "I'm the 'datatime' property")
967 datatime = property(getDatatime, "I'm the 'datatime' property")
951 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
968 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
952 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
969 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
953 channelIndexList = property(
970 channelIndexList = property(
954 getChannelIndexList, "I'm the 'channelIndexList' property.")
971 getChannelIndexList, "I'm the 'channelIndexList' property.")
955 noise = property(getNoise, "I'm the 'nHeights' property.")
972 noise = property(getNoise, "I'm the 'nHeights' property.")
956
973
957 ltctime = property(getltctime, "I'm the 'ltctime' property")
974 ltctime = property(getltctime, "I'm the 'ltctime' property")
958 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
975 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
959
976
960
977
961 class Correlation(JROData):
978 class Correlation(JROData):
962
979
963 noise = None
980 noise = None
964
981
965 SNR = None
982 SNR = None
966
983
967 #--------------------------------------------------
984 #--------------------------------------------------
968
985
969 mode = None
986 mode = None
970
987
971 split = False
988 split = False
972
989
973 data_cf = None
990 data_cf = None
974
991
975 lags = None
992 lags = None
976
993
977 lagRange = None
994 lagRange = None
978
995
979 pairsList = None
996 pairsList = None
980
997
981 normFactor = None
998 normFactor = None
982
999
983 #--------------------------------------------------
1000 #--------------------------------------------------
984
1001
985 # calculateVelocity = None
1002 # calculateVelocity = None
986
1003
987 nLags = None
1004 nLags = None
988
1005
989 nPairs = None
1006 nPairs = None
990
1007
991 nAvg = None
1008 nAvg = None
992
1009
993 def __init__(self):
1010 def __init__(self):
994 '''
1011 '''
995 Constructor
1012 Constructor
996 '''
1013 '''
997 self.radarControllerHeaderObj = RadarControllerHeader()
1014 self.radarControllerHeaderObj = RadarControllerHeader()
998
1015
999 self.systemHeaderObj = SystemHeader()
1016 self.systemHeaderObj = SystemHeader()
1000
1017
1001 self.type = "Correlation"
1018 self.type = "Correlation"
1002
1019
1003 self.data = None
1020 self.data = None
1004
1021
1005 self.dtype = None
1022 self.dtype = None
1006
1023
1007 self.nProfiles = None
1024 self.nProfiles = None
1008
1025
1009 self.heightList = None
1026 self.heightList = None
1010
1027
1011 self.channelList = None
1028 self.channelList = None
1012
1029
1013 self.flagNoData = True
1030 self.flagNoData = True
1014
1031
1015 self.flagDiscontinuousBlock = False
1032 self.flagDiscontinuousBlock = False
1016
1033
1017 self.utctime = None
1034 self.utctime = None
1018
1035
1019 self.timeZone = None
1036 self.timeZone = None
1020
1037
1021 self.dstFlag = None
1038 self.dstFlag = None
1022
1039
1023 self.errorCount = None
1040 self.errorCount = None
1024
1041
1025 self.blocksize = None
1042 self.blocksize = None
1026
1043
1027 self.flagDecodeData = False # asumo q la data no esta decodificada
1044 self.flagDecodeData = False # asumo q la data no esta decodificada
1028
1045
1029 self.flagDeflipData = False # asumo q la data no esta sin flip
1046 self.flagDeflipData = False # asumo q la data no esta sin flip
1030
1047
1031 self.pairsList = None
1048 self.pairsList = None
1032
1049
1033 self.nPoints = None
1050 self.nPoints = None
1034
1051
1035 def getPairsList(self):
1052 def getPairsList(self):
1036
1053
1037 return self.pairsList
1054 return self.pairsList
1038
1055
1039 def getNoise(self, mode=2):
1056 def getNoise(self, mode=2):
1040
1057
1041 indR = numpy.where(self.lagR == 0)[0][0]
1058 indR = numpy.where(self.lagR == 0)[0][0]
1042 indT = numpy.where(self.lagT == 0)[0][0]
1059 indT = numpy.where(self.lagT == 0)[0][0]
1043
1060
1044 jspectra0 = self.data_corr[:, :, indR, :]
1061 jspectra0 = self.data_corr[:, :, indR, :]
1045 jspectra = copy.copy(jspectra0)
1062 jspectra = copy.copy(jspectra0)
1046
1063
1047 num_chan = jspectra.shape[0]
1064 num_chan = jspectra.shape[0]
1048 num_hei = jspectra.shape[2]
1065 num_hei = jspectra.shape[2]
1049
1066
1050 freq_dc = jspectra.shape[1] / 2
1067 freq_dc = jspectra.shape[1] / 2
1051 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
1068 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
1052
1069
1053 if ind_vel[0] < 0:
1070 if ind_vel[0] < 0:
1054 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
1071 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
1055
1072
1056 if mode == 1:
1073 if mode == 1:
1057 jspectra[:, freq_dc, :] = (
1074 jspectra[:, freq_dc, :] = (
1058 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
1075 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
1059
1076
1060 if mode == 2:
1077 if mode == 2:
1061
1078
1062 vel = numpy.array([-2, -1, 1, 2])
1079 vel = numpy.array([-2, -1, 1, 2])
1063 xx = numpy.zeros([4, 4])
1080 xx = numpy.zeros([4, 4])
1064
1081
1065 for fil in range(4):
1082 for fil in range(4):
1066 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
1083 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
1067
1084
1068 xx_inv = numpy.linalg.inv(xx)
1085 xx_inv = numpy.linalg.inv(xx)
1069 xx_aux = xx_inv[0, :]
1086 xx_aux = xx_inv[0, :]
1070
1087
1071 for ich in range(num_chan):
1088 for ich in range(num_chan):
1072 yy = jspectra[ich, ind_vel, :]
1089 yy = jspectra[ich, ind_vel, :]
1073 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
1090 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
1074
1091
1075 junkid = jspectra[ich, freq_dc, :] <= 0
1092 junkid = jspectra[ich, freq_dc, :] <= 0
1076 cjunkid = sum(junkid)
1093 cjunkid = sum(junkid)
1077
1094
1078 if cjunkid.any():
1095 if cjunkid.any():
1079 jspectra[ich, freq_dc, junkid.nonzero()] = (
1096 jspectra[ich, freq_dc, junkid.nonzero()] = (
1080 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
1097 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
1081
1098
1082 noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
1099 noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
1083
1100
1084 return noise
1101 return noise
1085
1102
1086 def getTimeInterval(self):
1103 def getTimeInterval(self):
1087
1104
1088 timeInterval = self.ippSeconds * self.nCohInt * self.nProfiles
1105 timeInterval = self.ippSeconds * self.nCohInt * self.nProfiles
1089
1106
1090 return timeInterval
1107 return timeInterval
1091
1108
1092 def splitFunctions(self):
1109 def splitFunctions(self):
1093
1110
1094 pairsList = self.pairsList
1111 pairsList = self.pairsList
1095 ccf_pairs = []
1112 ccf_pairs = []
1096 acf_pairs = []
1113 acf_pairs = []
1097 ccf_ind = []
1114 ccf_ind = []
1098 acf_ind = []
1115 acf_ind = []
1099 for l in range(len(pairsList)):
1116 for l in range(len(pairsList)):
1100 chan0 = pairsList[l][0]
1117 chan0 = pairsList[l][0]
1101 chan1 = pairsList[l][1]
1118 chan1 = pairsList[l][1]
1102
1119
1103 # Obteniendo pares de Autocorrelacion
1120 # Obteniendo pares de Autocorrelacion
1104 if chan0 == chan1:
1121 if chan0 == chan1:
1105 acf_pairs.append(chan0)
1122 acf_pairs.append(chan0)
1106 acf_ind.append(l)
1123 acf_ind.append(l)
1107 else:
1124 else:
1108 ccf_pairs.append(pairsList[l])
1125 ccf_pairs.append(pairsList[l])
1109 ccf_ind.append(l)
1126 ccf_ind.append(l)
1110
1127
1111 data_acf = self.data_cf[acf_ind]
1128 data_acf = self.data_cf[acf_ind]
1112 data_ccf = self.data_cf[ccf_ind]
1129 data_ccf = self.data_cf[ccf_ind]
1113
1130
1114 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
1131 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
1115
1132
1116 def getNormFactor(self):
1133 def getNormFactor(self):
1117 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
1134 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
1118 acf_pairs = numpy.array(acf_pairs)
1135 acf_pairs = numpy.array(acf_pairs)
1119 normFactor = numpy.zeros((self.nPairs, self.nHeights))
1136 normFactor = numpy.zeros((self.nPairs, self.nHeights))
1120
1137
1121 for p in range(self.nPairs):
1138 for p in range(self.nPairs):
1122 pair = self.pairsList[p]
1139 pair = self.pairsList[p]
1123
1140
1124 ch0 = pair[0]
1141 ch0 = pair[0]
1125 ch1 = pair[1]
1142 ch1 = pair[1]
1126
1143
1127 ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
1144 ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
1128 ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
1145 ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
1129 normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
1146 normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
1130
1147
1131 return normFactor
1148 return normFactor
1132
1149
1133 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
1150 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
1134 normFactor = property(getNormFactor, "I'm the 'normFactor property'")
1151 normFactor = property(getNormFactor, "I'm the 'normFactor property'")
1135
1152
1136
1153
1137 class Parameters(Spectra):
1154 class Parameters(Spectra):
1138
1155
1139 experimentInfo = None # Information about the experiment
1156 experimentInfo = None # Information about the experiment
1140
1157
1141 # Information from previous data
1158 # Information from previous data
1142
1159
1143 inputUnit = None # Type of data to be processed
1160 inputUnit = None # Type of data to be processed
1144
1161
1145 operation = None # Type of operation to parametrize
1162 operation = None # Type of operation to parametrize
1146
1163
1147 # normFactor = None #Normalization Factor
1164 # normFactor = None #Normalization Factor
1148
1165
1149 groupList = None # List of Pairs, Groups, etc
1166 groupList = None # List of Pairs, Groups, etc
1150
1167
1151 # Parameters
1168 # Parameters
1152
1169
1153 data_param = None # Parameters obtained
1170 data_param = None # Parameters obtained
1154
1171
1155 data_pre = None # Data Pre Parametrization
1172 data_pre = None # Data Pre Parametrization
1156
1173
1157 data_SNR = None # Signal to Noise Ratio
1174 data_SNR = None # Signal to Noise Ratio
1158
1175
1159 # heightRange = None #Heights
1176 # heightRange = None #Heights
1160
1177
1161 abscissaList = None # Abscissa, can be velocities, lags or time
1178 abscissaList = None # Abscissa, can be velocities, lags or time
1162
1179
1163 # noise = None #Noise Potency
1180 # noise = None #Noise Potency
1164
1181
1165 utctimeInit = None # Initial UTC time
1182 utctimeInit = None # Initial UTC time
1166
1183
1167 paramInterval = None # Time interval to calculate Parameters in seconds
1184 paramInterval = None # Time interval to calculate Parameters in seconds
1168
1185
1169 useLocalTime = True
1186 useLocalTime = True
1170
1187
1171 # Fitting
1188 # Fitting
1172
1189
1173 data_error = None # Error of the estimation
1190 data_error = None # Error of the estimation
1174
1191
1175 constants = None
1192 constants = None
1176
1193
1177 library = None
1194 library = None
1178
1195
1179 # Output signal
1196 # Output signal
1180
1197
1181 outputInterval = None # Time interval to calculate output signal in seconds
1198 outputInterval = None # Time interval to calculate output signal in seconds
1182
1199
1183 data_output = None # Out signal
1200 data_output = None # Out signal
1184
1201
1185 nAvg = None
1202 nAvg = None
1186
1203
1187 noise_estimation = None
1204 noise_estimation = None
1188
1205
1189 GauSPC = None # Fit gaussian SPC
1206 GauSPC = None # Fit gaussian SPC
1190
1207
1191 def __init__(self):
1208 def __init__(self):
1192 '''
1209 '''
1193 Constructor
1210 Constructor
1194 '''
1211 '''
1195 self.radarControllerHeaderObj = RadarControllerHeader()
1212 self.radarControllerHeaderObj = RadarControllerHeader()
1196
1213
1197 self.systemHeaderObj = SystemHeader()
1214 self.systemHeaderObj = SystemHeader()
1198
1215
1199 self.type = "Parameters"
1216 self.type = "Parameters"
1200
1217
1201 def getTimeRange1(self, interval):
1218 def getTimeRange1(self, interval):
1202
1219
1203 datatime = []
1220 datatime = []
1204
1221
1205 if self.useLocalTime:
1222 if self.useLocalTime:
1206 time1 = self.utctimeInit - self.timeZone * 60
1223 time1 = self.utctimeInit - self.timeZone * 60
1207 else:
1224 else:
1208 time1 = self.utctimeInit
1225 time1 = self.utctimeInit
1209
1226
1210 datatime.append(time1)
1227 datatime.append(time1)
1211 datatime.append(time1 + interval)
1228 datatime.append(time1 + interval)
1212 datatime = numpy.array(datatime)
1229 datatime = numpy.array(datatime)
1213
1230
1214 return datatime
1231 return datatime
1215
1232
1216 def getTimeInterval(self):
1233 def getTimeInterval(self):
1217
1234
1218 if hasattr(self, 'timeInterval1'):
1235 if hasattr(self, 'timeInterval1'):
1219 return self.timeInterval1
1236 return self.timeInterval1
1220 else:
1237 else:
1221 return self.paramInterval
1238 return self.paramInterval
1222
1239
1223 def setValue(self, value):
1240 def setValue(self, value):
1224
1241
1225 print("This property should not be initialized")
1242 print("This property should not be initialized")
1226
1243
1227 return
1244 return
1228
1245
1229 def getNoise(self):
1246 def getNoise(self):
1230
1247
1231 return self.spc_noise
1248 return self.spc_noise
1232
1249
1233 timeInterval = property(getTimeInterval)
1250 timeInterval = property(getTimeInterval)
1234 noise = property(getNoise, setValue, "I'm the 'Noise' property.") No newline at end of file
1251 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
Show file before | Show file after | Hide comments
@@ -1,2151 +1,2158
1 import os
1 import os
2 import datetime
2 import datetime
3 import numpy
3 import numpy
4 import inspect
4 import inspect
5 from .figure import Figure, isRealtime, isTimeInHourRange
5 from .figure import Figure, isRealtime, isTimeInHourRange
6 from .plotting_codes import *
6 from .plotting_codes import *
7
7 from schainpy.model.proc.jroproc_base import MPDecorator
8 from schainpy.utils import log
8
9
9 class FitGauPlot(Figure):
10 class FitGauPlot(Figure):
10
11
11 isConfig = None
12 isConfig = None
12 __nsubplots = None
13 __nsubplots = None
13
14
14 WIDTHPROF = None
15 WIDTHPROF = None
15 HEIGHTPROF = None
16 HEIGHTPROF = None
16 PREFIX = 'fitgau'
17 PREFIX = 'fitgau'
17
18
18 def __init__(self, **kwargs):
19 def __init__(self, **kwargs):
19 Figure.__init__(self, **kwargs)
20 Figure.__init__(self, **kwargs)
20 self.isConfig = False
21 self.isConfig = False
21 self.__nsubplots = 1
22 self.__nsubplots = 1
22
23
23 self.WIDTH = 250
24 self.WIDTH = 250
24 self.HEIGHT = 250
25 self.HEIGHT = 250
25 self.WIDTHPROF = 120
26 self.WIDTHPROF = 120
26 self.HEIGHTPROF = 0
27 self.HEIGHTPROF = 0
27 self.counter_imagwr = 0
28 self.counter_imagwr = 0
28
29
29 self.PLOT_CODE = SPEC_CODE
30 self.PLOT_CODE = SPEC_CODE
30
31
31 self.FTP_WEI = None
32 self.FTP_WEI = None
32 self.EXP_CODE = None
33 self.EXP_CODE = None
33 self.SUB_EXP_CODE = None
34 self.SUB_EXP_CODE = None
34 self.PLOT_POS = None
35 self.PLOT_POS = None
35
36
36 self.__xfilter_ena = False
37 self.__xfilter_ena = False
37 self.__yfilter_ena = False
38 self.__yfilter_ena = False
38
39
39 def getSubplots(self):
40 def getSubplots(self):
40
41
41 ncol = int(numpy.sqrt(self.nplots)+0.9)
42 ncol = int(numpy.sqrt(self.nplots)+0.9)
42 nrow = int(self.nplots*1./ncol + 0.9)
43 nrow = int(self.nplots*1./ncol + 0.9)
43
44
44 return nrow, ncol
45 return nrow, ncol
45
46
46 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
47 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
47
48
48 self.__showprofile = showprofile
49 self.__showprofile = showprofile
49 self.nplots = nplots
50 self.nplots = nplots
50
51
51 ncolspan = 1
52 ncolspan = 1
52 colspan = 1
53 colspan = 1
53 if showprofile:
54 if showprofile:
54 ncolspan = 3
55 ncolspan = 3
55 colspan = 2
56 colspan = 2
56 self.__nsubplots = 2
57 self.__nsubplots = 2
57
58
58 self.createFigure(id = id,
59 self.createFigure(id = id,
59 wintitle = wintitle,
60 wintitle = wintitle,
60 widthplot = self.WIDTH + self.WIDTHPROF,
61 widthplot = self.WIDTH + self.WIDTHPROF,
61 heightplot = self.HEIGHT + self.HEIGHTPROF,
62 heightplot = self.HEIGHT + self.HEIGHTPROF,
62 show=show)
63 show=show)
63
64
64 nrow, ncol = self.getSubplots()
65 nrow, ncol = self.getSubplots()
65
66
66 counter = 0
67 counter = 0
67 for y in range(nrow):
68 for y in range(nrow):
68 for x in range(ncol):
69 for x in range(ncol):
69
70
70 if counter >= self.nplots:
71 if counter >= self.nplots:
71 break
72 break
72
73
73 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
74 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
74
75
75 if showprofile:
76 if showprofile:
76 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
77 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
77
78
78 counter += 1
79 counter += 1
79
80
80 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
81 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
81 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
82 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
82 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
83 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
83 server=None, folder=None, username=None, password=None,
84 server=None, folder=None, username=None, password=None,
84 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
85 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
85 xaxis="frequency", colormap='jet', normFactor=None , GauSelector = 1):
86 xaxis="frequency", colormap='jet', normFactor=None , GauSelector = 1):
86
87
87 """
88 """
88
89
89 Input:
90 Input:
90 dataOut :
91 dataOut :
91 id :
92 id :
92 wintitle :
93 wintitle :
93 channelList :
94 channelList :
94 showProfile :
95 showProfile :
95 xmin : None,
96 xmin : None,
96 xmax : None,
97 xmax : None,
97 ymin : None,
98 ymin : None,
98 ymax : None,
99 ymax : None,
99 zmin : None,
100 zmin : None,
100 zmax : None
101 zmax : None
101 """
102 """
102 if realtime:
103 if realtime:
103 if not(isRealtime(utcdatatime = dataOut.utctime)):
104 if not(isRealtime(utcdatatime = dataOut.utctime)):
104 print('Skipping this plot function')
105 print('Skipping this plot function')
105 return
106 return
106
107
107 if channelList == None:
108 if channelList == None:
108 channelIndexList = dataOut.channelIndexList
109 channelIndexList = dataOut.channelIndexList
109 else:
110 else:
110 channelIndexList = []
111 channelIndexList = []
111 for channel in channelList:
112 for channel in channelList:
112 if channel not in dataOut.channelList:
113 if channel not in dataOut.channelList:
113 raise ValueError("Channel %d is not in dataOut.channelList" %channel)
114 raise ValueError("Channel %d is not in dataOut.channelList" %channel)
114 channelIndexList.append(dataOut.channelList.index(channel))
115 channelIndexList.append(dataOut.channelList.index(channel))
115
116
116 # if normFactor is None:
117 # if normFactor is None:
117 # factor = dataOut.normFactor
118 # factor = dataOut.normFactor
118 # else:
119 # else:
119 # factor = normFactor
120 # factor = normFactor
120 if xaxis == "frequency":
121 if xaxis == "frequency":
121 x = dataOut.spc_range[0]
122 x = dataOut.spc_range[0]
122 xlabel = "Frequency (kHz)"
123 xlabel = "Frequency (kHz)"
123
124
124 elif xaxis == "time":
125 elif xaxis == "time":
125 x = dataOut.spc_range[1]
126 x = dataOut.spc_range[1]
126 xlabel = "Time (ms)"
127 xlabel = "Time (ms)"
127
128
128 else:
129 else:
129 x = dataOut.spc_range[2]
130 x = dataOut.spc_range[2]
130 xlabel = "Velocity (m/s)"
131 xlabel = "Velocity (m/s)"
131
132
132 ylabel = "Range (Km)"
133 ylabel = "Range (Km)"
133
134
134 y = dataOut.getHeiRange()
135 y = dataOut.getHeiRange()
135
136
136 z = dataOut.GauSPC[:,GauSelector,:,:] #GauSelector] #dataOut.data_spc/factor
137 z = dataOut.GauSPC[:,GauSelector,:,:] #GauSelector] #dataOut.data_spc/factor
137 print('GausSPC', z[0,32,10:40])
138 print('GausSPC', z[0,32,10:40])
138 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
139 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
139 zdB = 10*numpy.log10(z)
140 zdB = 10*numpy.log10(z)
140
141
141 avg = numpy.average(z, axis=1)
142 avg = numpy.average(z, axis=1)
142 avgdB = 10*numpy.log10(avg)
143 avgdB = 10*numpy.log10(avg)
143
144
144 noise = dataOut.spc_noise
145 noise = dataOut.spc_noise
145 noisedB = 10*numpy.log10(noise)
146 noisedB = 10*numpy.log10(noise)
146
147
147 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
148 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
148 title = wintitle + " Spectra"
149 title = wintitle + " Spectra"
149 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
150 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
150 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
151 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
151
152
152 if not self.isConfig:
153 if not self.isConfig:
153
154
154 nplots = len(channelIndexList)
155 nplots = len(channelIndexList)
155
156
156 self.setup(id=id,
157 self.setup(id=id,
157 nplots=nplots,
158 nplots=nplots,
158 wintitle=wintitle,
159 wintitle=wintitle,
159 showprofile=showprofile,
160 showprofile=showprofile,
160 show=show)
161 show=show)
161
162
162 if xmin == None: xmin = numpy.nanmin(x)
163 if xmin == None: xmin = numpy.nanmin(x)
163 if xmax == None: xmax = numpy.nanmax(x)
164 if xmax == None: xmax = numpy.nanmax(x)
164 if ymin == None: ymin = numpy.nanmin(y)
165 if ymin == None: ymin = numpy.nanmin(y)
165 if ymax == None: ymax = numpy.nanmax(y)
166 if ymax == None: ymax = numpy.nanmax(y)
166 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
167 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
167 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
168 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
168
169
169 self.FTP_WEI = ftp_wei
170 self.FTP_WEI = ftp_wei
170 self.EXP_CODE = exp_code
171 self.EXP_CODE = exp_code
171 self.SUB_EXP_CODE = sub_exp_code
172 self.SUB_EXP_CODE = sub_exp_code
172 self.PLOT_POS = plot_pos
173 self.PLOT_POS = plot_pos
173
174
174 self.isConfig = True
175 self.isConfig = True
175
176
176 self.setWinTitle(title)
177 self.setWinTitle(title)
177
178
178 for i in range(self.nplots):
179 for i in range(self.nplots):
179 index = channelIndexList[i]
180 index = channelIndexList[i]
180 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
181 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
181 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[index], noisedB[index], str_datetime)
182 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[index], noisedB[index], str_datetime)
182 if len(dataOut.beam.codeList) != 0:
183 if len(dataOut.beam.codeList) != 0:
183 title = "Ch%d:%4.2fdB,%2.2f,%2.2f:%s" %(dataOut.channelList[index], noisedB[index], dataOut.beam.azimuthList[index], dataOut.beam.zenithList[index], str_datetime)
184 title = "Ch%d:%4.2fdB,%2.2f,%2.2f:%s" %(dataOut.channelList[index], noisedB[index], dataOut.beam.azimuthList[index], dataOut.beam.zenithList[index], str_datetime)
184
185
185 axes = self.axesList[i*self.__nsubplots]
186 axes = self.axesList[i*self.__nsubplots]
186 axes.pcolor(x, y, zdB[index,:,:],
187 axes.pcolor(x, y, zdB[index,:,:],
187 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
188 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
188 xlabel=xlabel, ylabel=ylabel, title=title, colormap=colormap,
189 xlabel=xlabel, ylabel=ylabel, title=title, colormap=colormap,
189 ticksize=9, cblabel='')
190 ticksize=9, cblabel='')
190
191
191 if self.__showprofile:
192 if self.__showprofile:
192 axes = self.axesList[i*self.__nsubplots +1]
193 axes = self.axesList[i*self.__nsubplots +1]
193 axes.pline(avgdB[index,:], y,
194 axes.pline(avgdB[index,:], y,
194 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
195 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
195 xlabel='dB', ylabel='', title='',
196 xlabel='dB', ylabel='', title='',
196 ytick_visible=False,
197 ytick_visible=False,
197 grid='x')
198 grid='x')
198
199
199 noiseline = numpy.repeat(noisedB[index], len(y))
200 noiseline = numpy.repeat(noisedB[index], len(y))
200 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
201 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
201
202
202 self.draw()
203 self.draw()
203
204
204 if figfile == None:
205 if figfile == None:
205 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
206 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
206 name = str_datetime
207 name = str_datetime
207 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
208 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
208 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
209 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
209 figfile = self.getFilename(name)
210 figfile = self.getFilename(name)
210
211
211 self.save(figpath=figpath,
212 self.save(figpath=figpath,
212 figfile=figfile,
213 figfile=figfile,
213 save=save,
214 save=save,
214 ftp=ftp,
215 ftp=ftp,
215 wr_period=wr_period,
216 wr_period=wr_period,
216 thisDatetime=thisDatetime)
217 thisDatetime=thisDatetime)
217
218
218
219
219
220
220 class MomentsPlot(Figure):
221 class MomentsPlot(Figure):
221
222
222 isConfig = None
223 isConfig = None
223 __nsubplots = None
224 __nsubplots = None
224
225
225 WIDTHPROF = None
226 WIDTHPROF = None
226 HEIGHTPROF = None
227 HEIGHTPROF = None
227 PREFIX = 'prm'
228 PREFIX = 'prm'
228 def __init__(self, **kwargs):
229 def __init__(self):
229 Figure.__init__(self, **kwargs)
230 Figure.__init__(self)
230 self.isConfig = False
231 self.isConfig = False
231 self.__nsubplots = 1
232 self.__nsubplots = 1
232
233
233 self.WIDTH = 280
234 self.WIDTH = 280
234 self.HEIGHT = 250
235 self.HEIGHT = 250
235 self.WIDTHPROF = 120
236 self.WIDTHPROF = 120
236 self.HEIGHTPROF = 0
237 self.HEIGHTPROF = 0
237 self.counter_imagwr = 0
238 self.counter_imagwr = 0
238
239
239 self.PLOT_CODE = MOMENTS_CODE
240 self.PLOT_CODE = MOMENTS_CODE
240
241
241 self.FTP_WEI = None
242 self.FTP_WEI = None
242 self.EXP_CODE = None
243 self.EXP_CODE = None
243 self.SUB_EXP_CODE = None
244 self.SUB_EXP_CODE = None
244 self.PLOT_POS = None
245 self.PLOT_POS = None
245
246
246 def getSubplots(self):
247 def getSubplots(self):
247
248
248 ncol = int(numpy.sqrt(self.nplots)+0.9)
249 ncol = int(numpy.sqrt(self.nplots)+0.9)
249 nrow = int(self.nplots*1./ncol + 0.9)
250 nrow = int(self.nplots*1./ncol + 0.9)
250
251
251 return nrow, ncol
252 return nrow, ncol
252
253
253 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
254 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
254
255
255 self.__showprofile = showprofile
256 self.__showprofile = showprofile
256 self.nplots = nplots
257 self.nplots = nplots
257
258
258 ncolspan = 1
259 ncolspan = 1
259 colspan = 1
260 colspan = 1
260 if showprofile:
261 if showprofile:
261 ncolspan = 3
262 ncolspan = 3
262 colspan = 2
263 colspan = 2
263 self.__nsubplots = 2
264 self.__nsubplots = 2
264
265
265 self.createFigure(id = id,
266 self.createFigure(id = id,
266 wintitle = wintitle,
267 wintitle = wintitle,
267 widthplot = self.WIDTH + self.WIDTHPROF,
268 widthplot = self.WIDTH + self.WIDTHPROF,
268 heightplot = self.HEIGHT + self.HEIGHTPROF,
269 heightplot = self.HEIGHT + self.HEIGHTPROF,
269 show=show)
270 show=show)
270
271
271 nrow, ncol = self.getSubplots()
272 nrow, ncol = self.getSubplots()
272
273
273 counter = 0
274 counter = 0
274 for y in range(nrow):
275 for y in range(nrow):
275 for x in range(ncol):
276 for x in range(ncol):
276
277
277 if counter >= self.nplots:
278 if counter >= self.nplots:
278 break
279 break
279
280
280 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
281 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
281
282
282 if showprofile:
283 if showprofile:
283 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
284 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
284
285
285 counter += 1
286 counter += 1
286
287
287 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
288 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
288 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
289 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
289 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
290 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
290 server=None, folder=None, username=None, password=None,
291 server=None, folder=None, username=None, password=None,
291 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
292 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
292
293
293 """
294 """
294
295
295 Input:
296 Input:
296 dataOut :
297 dataOut :
297 id :
298 id :
298 wintitle :
299 wintitle :
299 channelList :
300 channelList :
300 showProfile :
301 showProfile :
301 xmin : None,
302 xmin : None,
302 xmax : None,
303 xmax : None,
303 ymin : None,
304 ymin : None,
304 ymax : None,
305 ymax : None,
305 zmin : None,
306 zmin : None,
306 zmax : None
307 zmax : None
307 """
308 """
308
309
309 if dataOut.flagNoData:
310 if dataOut.flagNoData:
310 return None
311 return None
311
312
312 if realtime:
313 if realtime:
313 if not(isRealtime(utcdatatime = dataOut.utctime)):
314 if not(isRealtime(utcdatatime = dataOut.utctime)):
314 print('Skipping this plot function')
315 print('Skipping this plot function')
315 return
316 return
316
317
317 if channelList == None:
318 if channelList == None:
318 channelIndexList = dataOut.channelIndexList
319 channelIndexList = dataOut.channelIndexList
319 else:
320 else:
320 channelIndexList = []
321 channelIndexList = []
321 for channel in channelList:
322 for channel in channelList:
322 if channel not in dataOut.channelList:
323 if channel not in dataOut.channelList:
323 raise ValueError("Channel %d is not in dataOut.channelList")
324 raise ValueError("Channel %d is not in dataOut.channelList")
324 channelIndexList.append(dataOut.channelList.index(channel))
325 channelIndexList.append(dataOut.channelList.index(channel))
325
326
326 factor = dataOut.normFactor
327 factor = dataOut.normFactor
327 x = dataOut.abscissaList
328 x = dataOut.abscissaList
328 y = dataOut.heightList
329 y = dataOut.heightList
329
330
330 z = dataOut.data_pre[channelIndexList,:,:]/factor
331 z = dataOut.data_pre[channelIndexList,:,:]/factor
331 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
332 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
332 avg = numpy.average(z, axis=1)
333 avg = numpy.average(z, axis=1)
333 noise = dataOut.noise/factor
334 noise = dataOut.noise/factor
334
335
335 zdB = 10*numpy.log10(z)
336 zdB = 10*numpy.log10(z)
336 avgdB = 10*numpy.log10(avg)
337 avgdB = 10*numpy.log10(avg)
337 noisedB = 10*numpy.log10(noise)
338 noisedB = 10*numpy.log10(noise)
338
339
339 #thisDatetime = dataOut.datatime
340 #thisDatetime = dataOut.datatime
340 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
341 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
341 title = wintitle + " Parameters"
342 title = wintitle + " Parameters"
342 xlabel = "Velocity (m/s)"
343 xlabel = "Velocity (m/s)"
343 ylabel = "Range (Km)"
344 ylabel = "Range (Km)"
344
345
345 update_figfile = False
346 update_figfile = False
346
347
347 if not self.isConfig:
348 if not self.isConfig:
348
349
349 nplots = len(channelIndexList)
350 nplots = len(channelIndexList)
350
351
351 self.setup(id=id,
352 self.setup(id=id,
352 nplots=nplots,
353 nplots=nplots,
353 wintitle=wintitle,
354 wintitle=wintitle,
354 showprofile=showprofile,
355 showprofile=showprofile,
355 show=show)
356 show=show)
356
357
357 if xmin == None: xmin = numpy.nanmin(x)
358 if xmin == None: xmin = numpy.nanmin(x)
358 if xmax == None: xmax = numpy.nanmax(x)
359 if xmax == None: xmax = numpy.nanmax(x)
359 if ymin == None: ymin = numpy.nanmin(y)
360 if ymin == None: ymin = numpy.nanmin(y)
360 if ymax == None: ymax = numpy.nanmax(y)
361 if ymax == None: ymax = numpy.nanmax(y)
361 if zmin == None: zmin = numpy.nanmin(avgdB)*0.9
362 if zmin == None: zmin = numpy.nanmin(avgdB)*0.9
362 if zmax == None: zmax = numpy.nanmax(avgdB)*0.9
363 if zmax == None: zmax = numpy.nanmax(avgdB)*0.9
363
364
364 self.FTP_WEI = ftp_wei
365 self.FTP_WEI = ftp_wei
365 self.EXP_CODE = exp_code
366 self.EXP_CODE = exp_code
366 self.SUB_EXP_CODE = sub_exp_code
367 self.SUB_EXP_CODE = sub_exp_code
367 self.PLOT_POS = plot_pos
368 self.PLOT_POS = plot_pos
368
369
369 self.isConfig = True
370 self.isConfig = True
370 update_figfile = True
371 update_figfile = True
371
372
372 self.setWinTitle(title)
373 self.setWinTitle(title)
373
374
374 for i in range(self.nplots):
375 for i in range(self.nplots):
375 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
376 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
376 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[i], noisedB[i], str_datetime)
377 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[i], noisedB[i], str_datetime)
377 axes = self.axesList[i*self.__nsubplots]
378 axes = self.axesList[i*self.__nsubplots]
378 axes.pcolor(x, y, zdB[i,:,:],
379 axes.pcolor(x, y, zdB[i,:,:],
379 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
380 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
380 xlabel=xlabel, ylabel=ylabel, title=title,
381 xlabel=xlabel, ylabel=ylabel, title=title,
381 ticksize=9, cblabel='')
382 ticksize=9, cblabel='')
382 #Mean Line
383 #Mean Line
383 mean = dataOut.data_param[i, 1, :]
384 mean = dataOut.data_param[i, 1, :]
384 axes.addpline(mean, y, idline=0, color="black", linestyle="solid", lw=1)
385 axes.addpline(mean, y, idline=0, color="black", linestyle="solid", lw=1)
385
386
386 if self.__showprofile:
387 if self.__showprofile:
387 axes = self.axesList[i*self.__nsubplots +1]
388 axes = self.axesList[i*self.__nsubplots +1]
388 axes.pline(avgdB[i], y,
389 axes.pline(avgdB[i], y,
389 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
390 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
390 xlabel='dB', ylabel='', title='',
391 xlabel='dB', ylabel='', title='',
391 ytick_visible=False,
392 ytick_visible=False,
392 grid='x')
393 grid='x')
393
394
394 noiseline = numpy.repeat(noisedB[i], len(y))
395 noiseline = numpy.repeat(noisedB[i], len(y))
395 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
396 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
396
397
397 self.draw()
398 self.draw()
398
399
399 self.save(figpath=figpath,
400 self.save(figpath=figpath,
400 figfile=figfile,
401 figfile=figfile,
401 save=save,
402 save=save,
402 ftp=ftp,
403 ftp=ftp,
403 wr_period=wr_period,
404 wr_period=wr_period,
404 thisDatetime=thisDatetime)
405 thisDatetime=thisDatetime)
405
406
406
407
407
408 class SkyMapPlot(Figure):
408 class SkyMapPlot(Figure):
409
409
410 __isConfig = None
410 __isConfig = None
411 __nsubplots = None
411 __nsubplots = None
412
412
413 WIDTHPROF = None
413 WIDTHPROF = None
414 HEIGHTPROF = None
414 HEIGHTPROF = None
415 PREFIX = 'mmap'
415 PREFIX = 'mmap'
416
416
417 def __init__(self, **kwargs):
417 def __init__(self, **kwargs):
418 Figure.__init__(self, **kwargs)
418 Figure.__init__(self, **kwargs)
419 self.isConfig = False
419 self.isConfig = False
420 self.__nsubplots = 1
420 self.__nsubplots = 1
421
421
422 # self.WIDTH = 280
422 # self.WIDTH = 280
423 # self.HEIGHT = 250
423 # self.HEIGHT = 250
424 self.WIDTH = 600
424 self.WIDTH = 600
425 self.HEIGHT = 600
425 self.HEIGHT = 600
426 self.WIDTHPROF = 120
426 self.WIDTHPROF = 120
427 self.HEIGHTPROF = 0
427 self.HEIGHTPROF = 0
428 self.counter_imagwr = 0
428 self.counter_imagwr = 0
429
429
430 self.PLOT_CODE = MSKYMAP_CODE
430 self.PLOT_CODE = MSKYMAP_CODE
431
431
432 self.FTP_WEI = None
432 self.FTP_WEI = None
433 self.EXP_CODE = None
433 self.EXP_CODE = None
434 self.SUB_EXP_CODE = None
434 self.SUB_EXP_CODE = None
435 self.PLOT_POS = None
435 self.PLOT_POS = None
436
436
437 def getSubplots(self):
437 def getSubplots(self):
438
438
439 ncol = int(numpy.sqrt(self.nplots)+0.9)
439 ncol = int(numpy.sqrt(self.nplots)+0.9)
440 nrow = int(self.nplots*1./ncol + 0.9)
440 nrow = int(self.nplots*1./ncol + 0.9)
441
441
442 return nrow, ncol
442 return nrow, ncol
443
443
444 def setup(self, id, nplots, wintitle, showprofile=False, show=True):
444 def setup(self, id, nplots, wintitle, showprofile=False, show=True):
445
445
446 self.__showprofile = showprofile
446 self.__showprofile = showprofile
447 self.nplots = nplots
447 self.nplots = nplots
448
448
449 ncolspan = 1
449 ncolspan = 1
450 colspan = 1
450 colspan = 1
451
451
452 self.createFigure(id = id,
452 self.createFigure(id = id,
453 wintitle = wintitle,
453 wintitle = wintitle,
454 widthplot = self.WIDTH, #+ self.WIDTHPROF,
454 widthplot = self.WIDTH, #+ self.WIDTHPROF,
455 heightplot = self.HEIGHT,# + self.HEIGHTPROF,
455 heightplot = self.HEIGHT,# + self.HEIGHTPROF,
456 show=show)
456 show=show)
457
457
458 nrow, ncol = 1,1
458 nrow, ncol = 1,1
459 counter = 0
459 counter = 0
460 x = 0
460 x = 0
461 y = 0
461 y = 0
462 self.addAxes(1, 1, 0, 0, 1, 1, True)
462 self.addAxes(1, 1, 0, 0, 1, 1, True)
463
463
464 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
464 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
465 tmin=0, tmax=24, timerange=None,
465 tmin=0, tmax=24, timerange=None,
466 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
466 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
467 server=None, folder=None, username=None, password=None,
467 server=None, folder=None, username=None, password=None,
468 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
468 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
469
469
470 """
470 """
471
471
472 Input:
472 Input:
473 dataOut :
473 dataOut :
474 id :
474 id :
475 wintitle :
475 wintitle :
476 channelList :
476 channelList :
477 showProfile :
477 showProfile :
478 xmin : None,
478 xmin : None,
479 xmax : None,
479 xmax : None,
480 ymin : None,
480 ymin : None,
481 ymax : None,
481 ymax : None,
482 zmin : None,
482 zmin : None,
483 zmax : None
483 zmax : None
484 """
484 """
485
485
486 arrayParameters = dataOut.data_param
486 arrayParameters = dataOut.data_param
487 error = arrayParameters[:,-1]
487 error = arrayParameters[:,-1]
488 indValid = numpy.where(error == 0)[0]
488 indValid = numpy.where(error == 0)[0]
489 finalMeteor = arrayParameters[indValid,:]
489 finalMeteor = arrayParameters[indValid,:]
490 finalAzimuth = finalMeteor[:,3]
490 finalAzimuth = finalMeteor[:,3]
491 finalZenith = finalMeteor[:,4]
491 finalZenith = finalMeteor[:,4]
492
492
493 x = finalAzimuth*numpy.pi/180
493 x = finalAzimuth*numpy.pi/180
494 y = finalZenith
494 y = finalZenith
495 x1 = [dataOut.ltctime, dataOut.ltctime]
495 x1 = [dataOut.ltctime, dataOut.ltctime]
496
496
497 #thisDatetime = dataOut.datatime
497 #thisDatetime = dataOut.datatime
498 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
498 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
499 title = wintitle + " Parameters"
499 title = wintitle + " Parameters"
500 xlabel = "Zonal Zenith Angle (deg) "
500 xlabel = "Zonal Zenith Angle (deg) "
501 ylabel = "Meridional Zenith Angle (deg)"
501 ylabel = "Meridional Zenith Angle (deg)"
502 update_figfile = False
502 update_figfile = False
503
503
504 if not self.isConfig:
504 if not self.isConfig:
505
505
506 nplots = 1
506 nplots = 1
507
507
508 self.setup(id=id,
508 self.setup(id=id,
509 nplots=nplots,
509 nplots=nplots,
510 wintitle=wintitle,
510 wintitle=wintitle,
511 showprofile=showprofile,
511 showprofile=showprofile,
512 show=show)
512 show=show)
513
513
514 if self.xmin is None and self.xmax is None:
514 if self.xmin is None and self.xmax is None:
515 self.xmin, self.xmax = self.getTimeLim(x1, tmin, tmax, timerange)
515 self.xmin, self.xmax = self.getTimeLim(x1, tmin, tmax, timerange)
516
516
517 if timerange != None:
517 if timerange != None:
518 self.timerange = timerange
518 self.timerange = timerange
519 else:
519 else:
520 self.timerange = self.xmax - self.xmin
520 self.timerange = self.xmax - self.xmin
521
521
522 self.FTP_WEI = ftp_wei
522 self.FTP_WEI = ftp_wei
523 self.EXP_CODE = exp_code
523 self.EXP_CODE = exp_code
524 self.SUB_EXP_CODE = sub_exp_code
524 self.SUB_EXP_CODE = sub_exp_code
525 self.PLOT_POS = plot_pos
525 self.PLOT_POS = plot_pos
526 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
526 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
527 self.firstdate = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
527 self.firstdate = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
528 self.isConfig = True
528 self.isConfig = True
529 update_figfile = True
529 update_figfile = True
530
530
531 self.setWinTitle(title)
531 self.setWinTitle(title)
532
532
533 i = 0
533 i = 0
534 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
534 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
535
535
536 axes = self.axesList[i*self.__nsubplots]
536 axes = self.axesList[i*self.__nsubplots]
537 nevents = axes.x_buffer.shape[0] + x.shape[0]
537 nevents = axes.x_buffer.shape[0] + x.shape[0]
538 title = "Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n" %(self.firstdate,str_datetime,nevents)
538 title = "Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n" %(self.firstdate,str_datetime,nevents)
539 axes.polar(x, y,
539 axes.polar(x, y,
540 title=title, xlabel=xlabel, ylabel=ylabel,
540 title=title, xlabel=xlabel, ylabel=ylabel,
541 ticksize=9, cblabel='')
541 ticksize=9, cblabel='')
542
542
543 self.draw()
543 self.draw()
544
544
545 self.save(figpath=figpath,
545 self.save(figpath=figpath,
546 figfile=figfile,
546 figfile=figfile,
547 save=save,
547 save=save,
548 ftp=ftp,
548 ftp=ftp,
549 wr_period=wr_period,
549 wr_period=wr_period,
550 thisDatetime=thisDatetime,
550 thisDatetime=thisDatetime,
551 update_figfile=update_figfile)
551 update_figfile=update_figfile)
552
552
553 if dataOut.ltctime >= self.xmax:
553 if dataOut.ltctime >= self.xmax:
554 self.isConfigmagwr = wr_period
554 self.isConfigmagwr = wr_period
555 self.isConfig = False
555 self.isConfig = False
556 update_figfile = True
556 update_figfile = True
557 axes.__firsttime = True
557 axes.__firsttime = True
558 self.xmin += self.timerange
558 self.xmin += self.timerange
559 self.xmax += self.timerange
559 self.xmax += self.timerange
560
560
561
561
562
562
563
563
564 class WindProfilerPlot(Figure):
564 class WindProfilerPlot(Figure):
565
565
566 __isConfig = None
566 __isConfig = None
567 __nsubplots = None
567 __nsubplots = None
568
568
569 WIDTHPROF = None
569 WIDTHPROF = None
570 HEIGHTPROF = None
570 HEIGHTPROF = None
571 PREFIX = 'wind'
571 PREFIX = 'wind'
572
572
573 def __init__(self, **kwargs):
573 def __init__(self, **kwargs):
574 Figure.__init__(self, **kwargs)
574 Figure.__init__(self, **kwargs)
575 self.timerange = None
575 self.timerange = None
576 self.isConfig = False
576 self.isConfig = False
577 self.__nsubplots = 1
577 self.__nsubplots = 1
578
578
579 self.WIDTH = 800
579 self.WIDTH = 800
580 self.HEIGHT = 300
580 self.HEIGHT = 300
581 self.WIDTHPROF = 120
581 self.WIDTHPROF = 120
582 self.HEIGHTPROF = 0
582 self.HEIGHTPROF = 0
583 self.counter_imagwr = 0
583 self.counter_imagwr = 0
584
584
585 self.PLOT_CODE = WIND_CODE
585 self.PLOT_CODE = WIND_CODE
586
586
587 self.FTP_WEI = None
587 self.FTP_WEI = None
588 self.EXP_CODE = None
588 self.EXP_CODE = None
589 self.SUB_EXP_CODE = None
589 self.SUB_EXP_CODE = None
590 self.PLOT_POS = None
590 self.PLOT_POS = None
591 self.tmin = None
591 self.tmin = None
592 self.tmax = None
592 self.tmax = None
593
593
594 self.xmin = None
594 self.xmin = None
595 self.xmax = None
595 self.xmax = None
596
596
597 self.figfile = None
597 self.figfile = None
598
598
599 def getSubplots(self):
599 def getSubplots(self):
600
600
601 ncol = 1
601 ncol = 1
602 nrow = self.nplots
602 nrow = self.nplots
603
603
604 return nrow, ncol
604 return nrow, ncol
605
605
606 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
606 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
607
607
608 self.__showprofile = showprofile
608 self.__showprofile = showprofile
609 self.nplots = nplots
609 self.nplots = nplots
610
610
611 ncolspan = 1
611 ncolspan = 1
612 colspan = 1
612 colspan = 1
613
613
614 self.createFigure(id = id,
614 self.createFigure(id = id,
615 wintitle = wintitle,
615 wintitle = wintitle,
616 widthplot = self.WIDTH + self.WIDTHPROF,
616 widthplot = self.WIDTH + self.WIDTHPROF,
617 heightplot = self.HEIGHT + self.HEIGHTPROF,
617 heightplot = self.HEIGHT + self.HEIGHTPROF,
618 show=show)
618 show=show)
619
619
620 nrow, ncol = self.getSubplots()
620 nrow, ncol = self.getSubplots()
621
621
622 counter = 0
622 counter = 0
623 for y in range(nrow):
623 for y in range(nrow):
624 if counter >= self.nplots:
624 if counter >= self.nplots:
625 break
625 break
626
626
627 self.addAxes(nrow, ncol*ncolspan, y, 0, colspan, 1)
627 self.addAxes(nrow, ncol*ncolspan, y, 0, colspan, 1)
628 counter += 1
628 counter += 1
629
629
630 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='False',
630 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='False',
631 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
631 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
632 zmax_ver = None, zmin_ver = None, SNRmin = None, SNRmax = None,
632 zmax_ver = None, zmin_ver = None, SNRmin = None, SNRmax = None,
633 timerange=None, SNRthresh = None,
633 timerange=None, SNRthresh = None,
634 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
634 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
635 server=None, folder=None, username=None, password=None,
635 server=None, folder=None, username=None, password=None,
636 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
636 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
637 """
637 """
638
638
639 Input:
639 Input:
640 dataOut :
640 dataOut :
641 id :
641 id :
642 wintitle :
642 wintitle :
643 channelList :
643 channelList :
644 showProfile :
644 showProfile :
645 xmin : None,
645 xmin : None,
646 xmax : None,
646 xmax : None,
647 ymin : None,
647 ymin : None,
648 ymax : None,
648 ymax : None,
649 zmin : None,
649 zmin : None,
650 zmax : None
650 zmax : None
651 """
651 """
652
652
653 # if timerange is not None:
653 # if timerange is not None:
654 # self.timerange = timerange
654 # self.timerange = timerange
655 #
655 #
656 # tmin = None
656 # tmin = None
657 # tmax = None
657 # tmax = None
658
658
659 x = dataOut.getTimeRange1(dataOut.paramInterval)
659 x = dataOut.getTimeRange1(dataOut.paramInterval)
660 y = dataOut.heightList
660 y = dataOut.heightList
661 z = dataOut.data_output.copy()
661 z = dataOut.data_output.copy()
662 nplots = z.shape[0] #Number of wind dimensions estimated
662 nplots = z.shape[0] #Number of wind dimensions estimated
663 nplotsw = nplots
663 nplotsw = nplots
664
664
665
665
666 #If there is a SNR function defined
666 #If there is a SNR function defined
667 if dataOut.data_SNR is not None:
667 if dataOut.data_SNR is not None:
668 nplots += 1
668 nplots += 1
669 SNR = dataOut.data_SNR
669 SNR = dataOut.data_SNR
670 SNRavg = numpy.average(SNR, axis=0)
670 SNRavg = numpy.average(SNR, axis=0)
671
671
672 SNRdB = 10*numpy.log10(SNR)
672 SNRdB = 10*numpy.log10(SNR)
673 SNRavgdB = 10*numpy.log10(SNRavg)
673 SNRavgdB = 10*numpy.log10(SNRavg)
674
674
675 if SNRthresh == None: SNRthresh = -5.0
675 if SNRthresh == None: SNRthresh = -5.0
676 ind = numpy.where(SNRavg < 10**(SNRthresh/10))[0]
676 ind = numpy.where(SNRavg < 10**(SNRthresh/10))[0]
677
677
678 for i in range(nplotsw):
678 for i in range(nplotsw):
679 z[i,ind] = numpy.nan
679 z[i,ind] = numpy.nan
680
680
681 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
681 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
682 #thisDatetime = datetime.datetime.now()
682 #thisDatetime = datetime.datetime.now()
683 title = wintitle + "Wind"
683 title = wintitle + "Wind"
684 xlabel = ""
684 xlabel = ""
685 ylabel = "Height (km)"
685 ylabel = "Height (km)"
686 update_figfile = False
686 update_figfile = False
687
687
688 if not self.isConfig:
688 if not self.isConfig:
689
689
690 self.setup(id=id,
690 self.setup(id=id,
691 nplots=nplots,
691 nplots=nplots,
692 wintitle=wintitle,
692 wintitle=wintitle,
693 showprofile=showprofile,
693 showprofile=showprofile,
694 show=show)
694 show=show)
695
695
696 if timerange is not None:
696 if timerange is not None:
697 self.timerange = timerange
697 self.timerange = timerange
698
698
699 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
699 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
700
700
701 if ymin == None: ymin = numpy.nanmin(y)
701 if ymin == None: ymin = numpy.nanmin(y)
702 if ymax == None: ymax = numpy.nanmax(y)
702 if ymax == None: ymax = numpy.nanmax(y)
703
703
704 if zmax == None: zmax = numpy.nanmax(abs(z[list(range(2)),:]))
704 if zmax == None: zmax = numpy.nanmax(abs(z[list(range(2)),:]))
705 #if numpy.isnan(zmax): zmax = 50
705 #if numpy.isnan(zmax): zmax = 50
706 if zmin == None: zmin = -zmax
706 if zmin == None: zmin = -zmax
707
707
708 if nplotsw == 3:
708 if nplotsw == 3:
709 if zmax_ver == None: zmax_ver = numpy.nanmax(abs(z[2,:]))
709 if zmax_ver == None: zmax_ver = numpy.nanmax(abs(z[2,:]))
710 if zmin_ver == None: zmin_ver = -zmax_ver
710 if zmin_ver == None: zmin_ver = -zmax_ver
711
711
712 if dataOut.data_SNR is not None:
712 if dataOut.data_SNR is not None:
713 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
713 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
714 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
714 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
715
715
716
716
717 self.FTP_WEI = ftp_wei
717 self.FTP_WEI = ftp_wei
718 self.EXP_CODE = exp_code
718 self.EXP_CODE = exp_code
719 self.SUB_EXP_CODE = sub_exp_code
719 self.SUB_EXP_CODE = sub_exp_code
720 self.PLOT_POS = plot_pos
720 self.PLOT_POS = plot_pos
721
721
722 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
722 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
723 self.isConfig = True
723 self.isConfig = True
724 self.figfile = figfile
724 self.figfile = figfile
725 update_figfile = True
725 update_figfile = True
726
726
727 self.setWinTitle(title)
727 self.setWinTitle(title)
728
728
729 if ((self.xmax - x[1]) < (x[1]-x[0])):
729 if ((self.xmax - x[1]) < (x[1]-x[0])):
730 x[1] = self.xmax
730 x[1] = self.xmax
731
731
732 strWind = ['Zonal', 'Meridional', 'Vertical']
732 strWind = ['Zonal', 'Meridional', 'Vertical']
733 strCb = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)']
733 strCb = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)']
734 zmaxVector = [zmax, zmax, zmax_ver]
734 zmaxVector = [zmax, zmax, zmax_ver]
735 zminVector = [zmin, zmin, zmin_ver]
735 zminVector = [zmin, zmin, zmin_ver]
736 windFactor = [1,1,100]
736 windFactor = [1,1,100]
737
737
738 for i in range(nplotsw):
738 for i in range(nplotsw):
739
739
740 title = "%s Wind: %s" %(strWind[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
740 title = "%s Wind: %s" %(strWind[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
741 axes = self.axesList[i*self.__nsubplots]
741 axes = self.axesList[i*self.__nsubplots]
742
742
743 z1 = z[i,:].reshape((1,-1))*windFactor[i]
743 z1 = z[i,:].reshape((1,-1))*windFactor[i]
744 #z1=numpy.ma.masked_where(z1==0.,z1)
744 #z1=numpy.ma.masked_where(z1==0.,z1)
745
745
746 axes.pcolorbuffer(x, y, z1,
746 axes.pcolorbuffer(x, y, z1,
747 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zminVector[i], zmax=zmaxVector[i],
747 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zminVector[i], zmax=zmaxVector[i],
748 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
748 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
749 ticksize=9, cblabel=strCb[i], cbsize="1%", colormap="seismic" )
749 ticksize=9, cblabel=strCb[i], cbsize="1%", colormap="seismic" )
750
750
751 if dataOut.data_SNR is not None:
751 if dataOut.data_SNR is not None:
752 i += 1
752 i += 1
753 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
753 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
754 axes = self.axesList[i*self.__nsubplots]
754 axes = self.axesList[i*self.__nsubplots]
755 SNRavgdB = SNRavgdB.reshape((1,-1))
755 SNRavgdB = SNRavgdB.reshape((1,-1))
756 axes.pcolorbuffer(x, y, SNRavgdB,
756 axes.pcolorbuffer(x, y, SNRavgdB,
757 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
757 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
758 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
758 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
759 ticksize=9, cblabel='', cbsize="1%", colormap="jet")
759 ticksize=9, cblabel='', cbsize="1%", colormap="jet")
760
760
761 self.draw()
761 self.draw()
762
762
763 self.save(figpath=figpath,
763 self.save(figpath=figpath,
764 figfile=figfile,
764 figfile=figfile,
765 save=save,
765 save=save,
766 ftp=ftp,
766 ftp=ftp,
767 wr_period=wr_period,
767 wr_period=wr_period,
768 thisDatetime=thisDatetime,
768 thisDatetime=thisDatetime,
769 update_figfile=update_figfile)
769 update_figfile=update_figfile)
770
770
771 if dataOut.ltctime + dataOut.paramInterval >= self.xmax:
771 if dataOut.ltctime + dataOut.paramInterval >= self.xmax:
772 self.counter_imagwr = wr_period
772 self.counter_imagwr = wr_period
773 self.isConfig = False
773 self.isConfig = False
774 update_figfile = True
774 update_figfile = True
775
775
776
776 @MPDecorator
777 class ParametersPlot(Figure):
777 class ParametersPlot(Figure):
778
778
779 __isConfig = None
779 __isConfig = None
780 __nsubplots = None
780 __nsubplots = None
781
781
782 WIDTHPROF = None
782 WIDTHPROF = None
783 HEIGHTPROF = None
783 HEIGHTPROF = None
784 PREFIX = 'param'
784 PREFIX = 'param'
785
785
786 nplots = None
786 nplots = None
787 nchan = None
787 nchan = None
788
788
789 def __init__(self, **kwargs):
789 def __init__(self):#, **kwargs):
790 Figure.__init__(self, **kwargs)
790 Figure.__init__(self)#, **kwargs)
791 self.timerange = None
791 self.timerange = None
792 self.isConfig = False
792 self.isConfig = False
793 self.__nsubplots = 1
793 self.__nsubplots = 1
794
794
795 self.WIDTH = 800
795 self.WIDTH = 800
796 self.HEIGHT = 180
796 self.HEIGHT = 180
797 self.WIDTHPROF = 120
797 self.WIDTHPROF = 120
798 self.HEIGHTPROF = 0
798 self.HEIGHTPROF = 0
799 self.counter_imagwr = 0
799 self.counter_imagwr = 0
800
800
801 self.PLOT_CODE = RTI_CODE
801 self.PLOT_CODE = RTI_CODE
802
802
803 self.FTP_WEI = None
803 self.FTP_WEI = None
804 self.EXP_CODE = None
804 self.EXP_CODE = None
805 self.SUB_EXP_CODE = None
805 self.SUB_EXP_CODE = None
806 self.PLOT_POS = None
806 self.PLOT_POS = None
807 self.tmin = None
807 self.tmin = None
808 self.tmax = None
808 self.tmax = None
809
809
810 self.xmin = None
810 self.xmin = None
811 self.xmax = None
811 self.xmax = None
812
812
813 self.figfile = None
813 self.figfile = None
814
814
815 def getSubplots(self):
815 def getSubplots(self):
816
816
817 ncol = 1
817 ncol = 1
818 nrow = self.nplots
818 nrow = self.nplots
819
819
820 return nrow, ncol
820 return nrow, ncol
821
821
822 def setup(self, id, nplots, wintitle, show=True):
822 def setup(self, id, nplots, wintitle, show=True):
823
823
824 self.nplots = nplots
824 self.nplots = nplots
825
825
826 ncolspan = 1
826 ncolspan = 1
827 colspan = 1
827 colspan = 1
828
828
829 self.createFigure(id = id,
829 self.createFigure(id = id,
830 wintitle = wintitle,
830 wintitle = wintitle,
831 widthplot = self.WIDTH + self.WIDTHPROF,
831 widthplot = self.WIDTH + self.WIDTHPROF,
832 heightplot = self.HEIGHT + self.HEIGHTPROF,
832 heightplot = self.HEIGHT + self.HEIGHTPROF,
833 show=show)
833 show=show)
834
834
835 nrow, ncol = self.getSubplots()
835 nrow, ncol = self.getSubplots()
836
836
837 counter = 0
837 counter = 0
838 for y in range(nrow):
838 for y in range(nrow):
839 for x in range(ncol):
839 for x in range(ncol):
840
840
841 if counter >= self.nplots:
841 if counter >= self.nplots:
842 break
842 break
843
843
844 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
844 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
845
845
846 counter += 1
846 counter += 1
847
847
848 def run(self, dataOut, id, wintitle="", channelList=None, paramIndex = 0, colormap="jet",
848 def run(self, dataOut, id, wintitle="", channelList=None, paramIndex = 0, colormap="jet",
849 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None, timerange=None,
849 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None, timerange=None,
850 showSNR=False, SNRthresh = -numpy.inf, SNRmin=None, SNRmax=None,
850 showSNR=False, SNRthresh = -numpy.inf, SNRmin=None, SNRmax=None,
851 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
851 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
852 server=None, folder=None, username=None, password=None,
852 server=None, folder=None, username=None, password=None,
853 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, HEIGHT=None):
853 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, HEIGHT=None):
854 """
854 """
855
855
856 Input:
856 Input:
857 dataOut :
857 dataOut :
858 id :
858 id :
859 wintitle :
859 wintitle :
860 channelList :
860 channelList :
861 showProfile :
861 showProfile :
862 xmin : None,
862 xmin : None,
863 xmax : None,
863 xmax : None,
864 ymin : None,
864 ymin : None,
865 ymax : None,
865 ymax : None,
866 zmin : None,
866 zmin : None,
867 zmax : None
867 zmax : None
868 """
868 """
869 if dataOut.flagNoData:
870 return dataOut
871
869
872
870 if HEIGHT is not None:
873 if HEIGHT is not None:
871 self.HEIGHT = HEIGHT
874 self.HEIGHT = HEIGHT
872
875
873
876
874 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
877 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
875 return
878 return
876
879
877 if channelList == None:
880 if channelList == None:
878 channelIndexList = list(range(dataOut.data_param.shape[0]))
881 channelIndexList = list(range(dataOut.data_param.shape[0]))
879 else:
882 else:
880 channelIndexList = []
883 channelIndexList = []
881 for channel in channelList:
884 for channel in channelList:
882 if channel not in dataOut.channelList:
885 if channel not in dataOut.channelList:
883 raise ValueError("Channel %d is not in dataOut.channelList")
886 raise ValueError("Channel %d is not in dataOut.channelList")
884 channelIndexList.append(dataOut.channelList.index(channel))
887 channelIndexList.append(dataOut.channelList.index(channel))
885
888
886 x = dataOut.getTimeRange1(dataOut.paramInterval)
889 x = dataOut.getTimeRange1(dataOut.paramInterval)
887 y = dataOut.getHeiRange()
890 y = dataOut.getHeiRange()
888
891
889 if dataOut.data_param.ndim == 3:
892 if dataOut.data_param.ndim == 3:
890 z = dataOut.data_param[channelIndexList,paramIndex,:]
893 z = dataOut.data_param[channelIndexList,paramIndex,:]
891 else:
894 else:
892 z = dataOut.data_param[channelIndexList,:]
895 z = dataOut.data_param[channelIndexList,:]
893
896
894 if showSNR:
897 if showSNR:
895 #SNR data
898 #SNR data
896 SNRarray = dataOut.data_SNR[channelIndexList,:]
899 SNRarray = dataOut.data_SNR[channelIndexList,:]
897 SNRdB = 10*numpy.log10(SNRarray)
900 SNRdB = 10*numpy.log10(SNRarray)
898 ind = numpy.where(SNRdB < SNRthresh)
901 ind = numpy.where(SNRdB < SNRthresh)
899 z[ind] = numpy.nan
902 z[ind] = numpy.nan
900
903
901 thisDatetime = dataOut.datatime
904 thisDatetime = dataOut.datatime
902 # thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
905 # thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
903 title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
906 title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
904 xlabel = ""
907 xlabel = ""
905 ylabel = "Range (Km)"
908 ylabel = "Range (Km)"
906
909
907 update_figfile = False
910 update_figfile = False
908
911
909 if not self.isConfig:
912 if not self.isConfig:
910
913
911 nchan = len(channelIndexList)
914 nchan = len(channelIndexList)
912 self.nchan = nchan
915 self.nchan = nchan
913 self.plotFact = 1
916 self.plotFact = 1
914 nplots = nchan
917 nplots = nchan
915
918
916 if showSNR:
919 if showSNR:
917 nplots = nchan*2
920 nplots = nchan*2
918 self.plotFact = 2
921 self.plotFact = 2
919 if SNRmin == None: SNRmin = numpy.nanmin(SNRdB)
922 if SNRmin == None: SNRmin = numpy.nanmin(SNRdB)
920 if SNRmax == None: SNRmax = numpy.nanmax(SNRdB)
923 if SNRmax == None: SNRmax = numpy.nanmax(SNRdB)
921
924
922 self.setup(id=id,
925 self.setup(id=id,
923 nplots=nplots,
926 nplots=nplots,
924 wintitle=wintitle,
927 wintitle=wintitle,
925 show=show)
928 show=show)
926
929
927 if timerange != None:
930 if timerange != None:
928 self.timerange = timerange
931 self.timerange = timerange
929
932
930 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
933 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
931
934
932 if ymin == None: ymin = numpy.nanmin(y)
935 if ymin == None: ymin = numpy.nanmin(y)
933 if ymax == None: ymax = numpy.nanmax(y)
936 if ymax == None: ymax = numpy.nanmax(y)
934 if zmin == None: zmin = numpy.nanmin(z)
937 if zmin == None: zmin = numpy.nanmin(z)
935 if zmax == None: zmax = numpy.nanmax(z)
938 if zmax == None: zmax = numpy.nanmax(z)
936
939
937 self.FTP_WEI = ftp_wei
940 self.FTP_WEI = ftp_wei
938 self.EXP_CODE = exp_code
941 self.EXP_CODE = exp_code
939 self.SUB_EXP_CODE = sub_exp_code
942 self.SUB_EXP_CODE = sub_exp_code
940 self.PLOT_POS = plot_pos
943 self.PLOT_POS = plot_pos
941
944
942 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
945 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
943 self.isConfig = True
946 self.isConfig = True
944 self.figfile = figfile
947 self.figfile = figfile
945 update_figfile = True
948 update_figfile = True
946
949
947 self.setWinTitle(title)
950 self.setWinTitle(title)
948
951
949 for i in range(self.nchan):
952 for i in range(self.nchan):
950 index = channelIndexList[i]
953 index = channelIndexList[i]
951 title = "Channel %d: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
954 title = "Channel %d: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
952 axes = self.axesList[i*self.plotFact]
955 axes = self.axesList[i*self.plotFact]
953 z1 = z[i,:].reshape((1,-1))
956 z1 = z[i,:].reshape((1,-1))
954 axes.pcolorbuffer(x, y, z1,
957 axes.pcolorbuffer(x, y, z1,
955 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
958 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
956 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
959 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
957 ticksize=9, cblabel='', cbsize="1%",colormap=colormap)
960 ticksize=9, cblabel='', cbsize="1%",colormap=colormap)
958
961
959 if showSNR:
962 if showSNR:
960 title = "Channel %d SNR: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
963 title = "Channel %d SNR: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
961 axes = self.axesList[i*self.plotFact + 1]
964 axes = self.axesList[i*self.plotFact + 1]
962 SNRdB1 = SNRdB[i,:].reshape((1,-1))
965 SNRdB1 = SNRdB[i,:].reshape((1,-1))
963 axes.pcolorbuffer(x, y, SNRdB1,
966 axes.pcolorbuffer(x, y, SNRdB1,
964 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
967 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
965 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
968 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
966 ticksize=9, cblabel='', cbsize="1%",colormap='jet')
969 ticksize=9, cblabel='', cbsize="1%",colormap='jet')
967
970
968
971
969 self.draw()
972 self.draw()
970
973
971 if dataOut.ltctime >= self.xmax:
974 if dataOut.ltctime >= self.xmax:
972 self.counter_imagwr = wr_period
975 self.counter_imagwr = wr_period
973 self.isConfig = False
976 self.isConfig = False
974 update_figfile = True
977 update_figfile = True
975
978
976 self.save(figpath=figpath,
979 self.save(figpath=figpath,
977 figfile=figfile,
980 figfile=figfile,
978 save=save,
981 save=save,
979 ftp=ftp,
982 ftp=ftp,
980 wr_period=wr_period,
983 wr_period=wr_period,
981 thisDatetime=thisDatetime,
984 thisDatetime=thisDatetime,
982 update_figfile=update_figfile)
985 update_figfile=update_figfile)
983
986
984
987 return dataOut
985
988 @MPDecorator
986 class Parameters1Plot(Figure):
989 class Parameters1Plot(Figure):
987
990
988 __isConfig = None
991 __isConfig = None
989 __nsubplots = None
992 __nsubplots = None
990
993
991 WIDTHPROF = None
994 WIDTHPROF = None
992 HEIGHTPROF = None
995 HEIGHTPROF = None
993 PREFIX = 'prm'
996 PREFIX = 'prm'
994
997
995 def __init__(self, **kwargs):
998 def __init__(self):
996 Figure.__init__(self, **kwargs)
999 Figure.__init__(self)
997 self.timerange = 2*60*60
1000 self.timerange = 2*60*60
998 self.isConfig = False
1001 self.isConfig = False
999 self.__nsubplots = 1
1002 self.__nsubplots = 1
1000
1003
1001 self.WIDTH = 800
1004 self.WIDTH = 800
1002 self.HEIGHT = 180
1005 self.HEIGHT = 180
1003 self.WIDTHPROF = 120
1006 self.WIDTHPROF = 120
1004 self.HEIGHTPROF = 0
1007 self.HEIGHTPROF = 0
1005 self.counter_imagwr = 0
1008 self.counter_imagwr = 0
1006
1009
1007 self.PLOT_CODE = PARMS_CODE
1010 self.PLOT_CODE = PARMS_CODE
1008
1011
1009 self.FTP_WEI = None
1012 self.FTP_WEI = None
1010 self.EXP_CODE = None
1013 self.EXP_CODE = None
1011 self.SUB_EXP_CODE = None
1014 self.SUB_EXP_CODE = None
1012 self.PLOT_POS = None
1015 self.PLOT_POS = None
1013 self.tmin = None
1016 self.tmin = None
1014 self.tmax = None
1017 self.tmax = None
1015
1018
1016 self.xmin = None
1019 self.xmin = None
1017 self.xmax = None
1020 self.xmax = None
1018
1021
1019 self.figfile = None
1022 self.figfile = None
1020
1023
1021 def getSubplots(self):
1024 def getSubplots(self):
1022
1025
1023 ncol = 1
1026 ncol = 1
1024 nrow = self.nplots
1027 nrow = self.nplots
1025
1028
1026 return nrow, ncol
1029 return nrow, ncol
1027
1030
1028 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1031 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1029
1032
1030 self.__showprofile = showprofile
1033 self.__showprofile = showprofile
1031 self.nplots = nplots
1034 self.nplots = nplots
1032
1035
1033 ncolspan = 1
1036 ncolspan = 1
1034 colspan = 1
1037 colspan = 1
1035
1038
1036 self.createFigure(id = id,
1039 self.createFigure(id = id,
1037 wintitle = wintitle,
1040 wintitle = wintitle,
1038 widthplot = self.WIDTH + self.WIDTHPROF,
1041 widthplot = self.WIDTH + self.WIDTHPROF,
1039 heightplot = self.HEIGHT + self.HEIGHTPROF,
1042 heightplot = self.HEIGHT + self.HEIGHTPROF,
1040 show=show)
1043 show=show)
1041
1044
1042 nrow, ncol = self.getSubplots()
1045 nrow, ncol = self.getSubplots()
1043
1046
1044 counter = 0
1047 counter = 0
1045 for y in range(nrow):
1048 for y in range(nrow):
1046 for x in range(ncol):
1049 for x in range(ncol):
1047
1050
1048 if counter >= self.nplots:
1051 if counter >= self.nplots:
1049 break
1052 break
1050
1053
1051 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1054 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1052
1055
1053 if showprofile:
1056 if showprofile:
1054 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
1057 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
1055
1058
1056 counter += 1
1059 counter += 1
1057
1060
1058 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
1061 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
1059 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,timerange=None,
1062 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,timerange=None,
1060 parameterIndex = None, onlyPositive = False,
1063 parameterIndex = None, onlyPositive = False,
1061 SNRthresh = -numpy.inf, SNR = True, SNRmin = None, SNRmax = None, onlySNR = False,
1064 SNRthresh = -numpy.inf, SNR = True, SNRmin = None, SNRmax = None, onlySNR = False,
1062 DOP = True,
1065 DOP = True,
1063 zlabel = "", parameterName = "", parameterObject = "data_param",
1066 zlabel = "", parameterName = "", parameterObject = "data_param",
1064 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
1067 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
1065 server=None, folder=None, username=None, password=None,
1068 server=None, folder=None, username=None, password=None,
1066 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1069 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1067 #print inspect.getargspec(self.run).args
1070 #print inspect.getargspec(self.run).args
1068 """
1071 """
1069
1072
1070 Input:
1073 Input:
1071 dataOut :
1074 dataOut :
1072 id :
1075 id :
1073 wintitle :
1076 wintitle :
1074 channelList :
1077 channelList :
1075 showProfile :
1078 showProfile :
1076 xmin : None,
1079 xmin : None,
1077 xmax : None,
1080 xmax : None,
1078 ymin : None,
1081 ymin : None,
1079 ymax : None,
1082 ymax : None,
1080 zmin : None,
1083 zmin : None,
1081 zmax : None
1084 zmax : None
1082 """
1085 """
1086 if dataOut.flagNoData:
1087 return dataOut
1083
1088
1084 data_param = getattr(dataOut, parameterObject)
1089 data_param = getattr(dataOut, parameterObject)
1085
1090
1086 if channelList == None:
1091 if channelList == None:
1087 channelIndexList = numpy.arange(data_param.shape[0])
1092 channelIndexList = numpy.arange(data_param.shape[0])
1088 else:
1093 else:
1089 channelIndexList = numpy.array(channelList)
1094 channelIndexList = numpy.array(channelList)
1090
1095
1091 nchan = len(channelIndexList) #Number of channels being plotted
1096 nchan = len(channelIndexList) #Number of channels being plotted
1092
1097
1093 if nchan < 1:
1098 if nchan < 1:
1094 return
1099 return
1095
1100
1096 nGraphsByChannel = 0
1101 nGraphsByChannel = 0
1097
1102
1098 if SNR:
1103 if SNR:
1099 nGraphsByChannel += 1
1104 nGraphsByChannel += 1
1100 if DOP:
1105 if DOP:
1101 nGraphsByChannel += 1
1106 nGraphsByChannel += 1
1102
1107
1103 if nGraphsByChannel < 1:
1108 if nGraphsByChannel < 1:
1104 return
1109 return
1105
1110
1106 nplots = nGraphsByChannel*nchan
1111 nplots = nGraphsByChannel*nchan
1107
1112
1108 if timerange is not None:
1113 if timerange is not None:
1109 self.timerange = timerange
1114 self.timerange = timerange
1110
1115
1111 #tmin = None
1116 #tmin = None
1112 #tmax = None
1117 #tmax = None
1113 if parameterIndex == None:
1118 if parameterIndex == None:
1114 parameterIndex = 1
1119 parameterIndex = 1
1115
1120
1116 x = dataOut.getTimeRange1(dataOut.paramInterval)
1121 x = dataOut.getTimeRange1(dataOut.paramInterval)
1117 y = dataOut.heightList
1122 y = dataOut.heightList
1118
1123
1119 if dataOut.data_param.ndim == 3:
1124 if dataOut.data_param.ndim == 3:
1120 z = dataOut.data_param[channelIndexList,parameterIndex,:]
1125 z = dataOut.data_param[channelIndexList,parameterIndex,:]
1121 else:
1126 else:
1122 z = dataOut.data_param[channelIndexList,:]
1127 z = dataOut.data_param[channelIndexList,:]
1123
1128
1124 if dataOut.data_SNR is not None:
1129 if dataOut.data_SNR is not None:
1125 if dataOut.data_SNR.ndim == 2:
1130 if dataOut.data_SNR.ndim == 2:
1126 SNRavg = numpy.average(dataOut.data_SNR, axis=0)
1131 SNRavg = numpy.average(dataOut.data_SNR, axis=0)
1127 else:
1132 else:
1128 SNRavg = dataOut.data_SNR
1133 SNRavg = dataOut.data_SNR
1129 SNRdB = 10*numpy.log10(SNRavg)
1134 SNRdB = 10*numpy.log10(SNRavg)
1130
1135
1131 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1136 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1132 title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
1137 title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
1133 xlabel = ""
1138 xlabel = ""
1134 ylabel = "Range (Km)"
1139 ylabel = "Range (Km)"
1135
1140
1136 if onlyPositive:
1141 if onlyPositive:
1137 colormap = "jet"
1142 colormap = "jet"
1138 zmin = 0
1143 zmin = 0
1139 else: colormap = "RdBu_r"
1144 else: colormap = "RdBu_r"
1140
1145
1141 if not self.isConfig:
1146 if not self.isConfig:
1142
1147
1143 self.setup(id=id,
1148 self.setup(id=id,
1144 nplots=nplots,
1149 nplots=nplots,
1145 wintitle=wintitle,
1150 wintitle=wintitle,
1146 showprofile=showprofile,
1151 showprofile=showprofile,
1147 show=show)
1152 show=show)
1148
1153
1149 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1154 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1150
1155
1151 if ymin == None: ymin = numpy.nanmin(y)
1156 if ymin == None: ymin = numpy.nanmin(y)
1152 if ymax == None: ymax = numpy.nanmax(y)
1157 if ymax == None: ymax = numpy.nanmax(y)
1153 if zmin == None: zmin = numpy.nanmin(z)
1158 if zmin == None: zmin = numpy.nanmin(z)
1154 if zmax == None: zmax = numpy.nanmax(z)
1159 if zmax == None: zmax = numpy.nanmax(z)
1155
1160
1156 if SNR:
1161 if SNR:
1157 if SNRmin == None: SNRmin = numpy.nanmin(SNRdB)
1162 if SNRmin == None: SNRmin = numpy.nanmin(SNRdB)
1158 if SNRmax == None: SNRmax = numpy.nanmax(SNRdB)
1163 if SNRmax == None: SNRmax = numpy.nanmax(SNRdB)
1159
1164
1160 self.FTP_WEI = ftp_wei
1165 self.FTP_WEI = ftp_wei
1161 self.EXP_CODE = exp_code
1166 self.EXP_CODE = exp_code
1162 self.SUB_EXP_CODE = sub_exp_code
1167 self.SUB_EXP_CODE = sub_exp_code
1163 self.PLOT_POS = plot_pos
1168 self.PLOT_POS = plot_pos
1164
1169
1165 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1170 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1166 self.isConfig = True
1171 self.isConfig = True
1167 self.figfile = figfile
1172 self.figfile = figfile
1168
1173
1169 self.setWinTitle(title)
1174 self.setWinTitle(title)
1170
1175
1171 if ((self.xmax - x[1]) < (x[1]-x[0])):
1176 if ((self.xmax - x[1]) < (x[1]-x[0])):
1172 x[1] = self.xmax
1177 x[1] = self.xmax
1173
1178
1174 for i in range(nchan):
1179 for i in range(nchan):
1175
1180
1176 if (SNR and not onlySNR): j = 2*i
1181 if (SNR and not onlySNR): j = 2*i
1177 else: j = i
1182 else: j = i
1178
1183
1179 j = nGraphsByChannel*i
1184 j = nGraphsByChannel*i
1180
1185
1181 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
1186 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
1182 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
1187 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
1183
1188
1184 if not onlySNR:
1189 if not onlySNR:
1185 axes = self.axesList[j*self.__nsubplots]
1190 axes = self.axesList[j*self.__nsubplots]
1186 z1 = z[i,:].reshape((1,-1))
1191 z1 = z[i,:].reshape((1,-1))
1187 axes.pcolorbuffer(x, y, z1,
1192 axes.pcolorbuffer(x, y, z1,
1188 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
1193 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
1189 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
1194 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
1190 ticksize=9, cblabel=zlabel, cbsize="1%")
1195 ticksize=9, cblabel=zlabel, cbsize="1%")
1191
1196
1192 if DOP:
1197 if DOP:
1193 title = "%s Channel %d: %s" %(parameterName, channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1198 title = "%s Channel %d: %s" %(parameterName, channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1194
1199
1195 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
1200 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
1196 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
1201 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
1197 axes = self.axesList[j]
1202 axes = self.axesList[j]
1198 z1 = z[i,:].reshape((1,-1))
1203 z1 = z[i,:].reshape((1,-1))
1199 axes.pcolorbuffer(x, y, z1,
1204 axes.pcolorbuffer(x, y, z1,
1200 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
1205 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
1201 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
1206 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
1202 ticksize=9, cblabel=zlabel, cbsize="1%")
1207 ticksize=9, cblabel=zlabel, cbsize="1%")
1203
1208
1204 if SNR:
1209 if SNR:
1205 title = "Channel %d Signal Noise Ratio (SNR): %s" %(channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1210 title = "Channel %d Signal Noise Ratio (SNR): %s" %(channelIndexList[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1206 axes = self.axesList[(j)*self.__nsubplots]
1211 axes = self.axesList[(j)*self.__nsubplots]
1207 if not onlySNR:
1212 if not onlySNR:
1208 axes = self.axesList[(j + 1)*self.__nsubplots]
1213 axes = self.axesList[(j + 1)*self.__nsubplots]
1209
1214
1210 axes = self.axesList[(j + nGraphsByChannel-1)]
1215 axes = self.axesList[(j + nGraphsByChannel-1)]
1211 z1 = SNRdB.reshape((1,-1))
1216 z1 = SNRdB.reshape((1,-1))
1212 axes.pcolorbuffer(x, y, z1,
1217 axes.pcolorbuffer(x, y, z1,
1213 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
1218 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
1214 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap="jet",
1219 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap="jet",
1215 ticksize=9, cblabel=zlabel, cbsize="1%")
1220 ticksize=9, cblabel=zlabel, cbsize="1%")
1216
1221
1217
1222
1218
1223
1219 self.draw()
1224 self.draw()
1220
1225
1221 if x[1] >= self.axesList[0].xmax:
1226 if x[1] >= self.axesList[0].xmax:
1222 self.counter_imagwr = wr_period
1227 self.counter_imagwr = wr_period
1223 self.isConfig = False
1228 self.isConfig = False
1224 self.figfile = None
1229 self.figfile = None
1225
1230
1226 self.save(figpath=figpath,
1231 self.save(figpath=figpath,
1227 figfile=figfile,
1232 figfile=figfile,
1228 save=save,
1233 save=save,
1229 ftp=ftp,
1234 ftp=ftp,
1230 wr_period=wr_period,
1235 wr_period=wr_period,
1231 thisDatetime=thisDatetime,
1236 thisDatetime=thisDatetime,
1232 update_figfile=False)
1237 update_figfile=False)
1238 return dataOut
1233
1239
1234 class SpectralFittingPlot(Figure):
1240 class SpectralFittingPlot(Figure):
1235
1241
1236 __isConfig = None
1242 __isConfig = None
1237 __nsubplots = None
1243 __nsubplots = None
1238
1244
1239 WIDTHPROF = None
1245 WIDTHPROF = None
1240 HEIGHTPROF = None
1246 HEIGHTPROF = None
1241 PREFIX = 'prm'
1247 PREFIX = 'prm'
1242
1248
1243
1249
1244 N = None
1250 N = None
1245 ippSeconds = None
1251 ippSeconds = None
1246
1252
1247 def __init__(self, **kwargs):
1253 def __init__(self, **kwargs):
1248 Figure.__init__(self, **kwargs)
1254 Figure.__init__(self, **kwargs)
1249 self.isConfig = False
1255 self.isConfig = False
1250 self.__nsubplots = 1
1256 self.__nsubplots = 1
1251
1257
1252 self.PLOT_CODE = SPECFIT_CODE
1258 self.PLOT_CODE = SPECFIT_CODE
1253
1259
1254 self.WIDTH = 450
1260 self.WIDTH = 450
1255 self.HEIGHT = 250
1261 self.HEIGHT = 250
1256 self.WIDTHPROF = 0
1262 self.WIDTHPROF = 0
1257 self.HEIGHTPROF = 0
1263 self.HEIGHTPROF = 0
1258
1264
1259 def getSubplots(self):
1265 def getSubplots(self):
1260
1266
1261 ncol = int(numpy.sqrt(self.nplots)+0.9)
1267 ncol = int(numpy.sqrt(self.nplots)+0.9)
1262 nrow = int(self.nplots*1./ncol + 0.9)
1268 nrow = int(self.nplots*1./ncol + 0.9)
1263
1269
1264 return nrow, ncol
1270 return nrow, ncol
1265
1271
1266 def setup(self, id, nplots, wintitle, showprofile=False, show=True):
1272 def setup(self, id, nplots, wintitle, showprofile=False, show=True):
1267
1273
1268 showprofile = False
1274 showprofile = False
1269 self.__showprofile = showprofile
1275 self.__showprofile = showprofile
1270 self.nplots = nplots
1276 self.nplots = nplots
1271
1277
1272 ncolspan = 5
1278 ncolspan = 5
1273 colspan = 4
1279 colspan = 4
1274 if showprofile:
1280 if showprofile:
1275 ncolspan = 5
1281 ncolspan = 5
1276 colspan = 4
1282 colspan = 4
1277 self.__nsubplots = 2
1283 self.__nsubplots = 2
1278
1284
1279 self.createFigure(id = id,
1285 self.createFigure(id = id,
1280 wintitle = wintitle,
1286 wintitle = wintitle,
1281 widthplot = self.WIDTH + self.WIDTHPROF,
1287 widthplot = self.WIDTH + self.WIDTHPROF,
1282 heightplot = self.HEIGHT + self.HEIGHTPROF,
1288 heightplot = self.HEIGHT + self.HEIGHTPROF,
1283 show=show)
1289 show=show)
1284
1290
1285 nrow, ncol = self.getSubplots()
1291 nrow, ncol = self.getSubplots()
1286
1292
1287 counter = 0
1293 counter = 0
1288 for y in range(nrow):
1294 for y in range(nrow):
1289 for x in range(ncol):
1295 for x in range(ncol):
1290
1296
1291 if counter >= self.nplots:
1297 if counter >= self.nplots:
1292 break
1298 break
1293
1299
1294 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1300 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1295
1301
1296 if showprofile:
1302 if showprofile:
1297 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
1303 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
1298
1304
1299 counter += 1
1305 counter += 1
1300
1306
1301 def run(self, dataOut, id, cutHeight=None, fit=False, wintitle="", channelList=None, showprofile=True,
1307 def run(self, dataOut, id, cutHeight=None, fit=False, wintitle="", channelList=None, showprofile=True,
1302 xmin=None, xmax=None, ymin=None, ymax=None,
1308 xmin=None, xmax=None, ymin=None, ymax=None,
1303 save=False, figpath='./', figfile=None, show=True):
1309 save=False, figpath='./', figfile=None, show=True):
1304
1310
1305 """
1311 """
1306
1312
1307 Input:
1313 Input:
1308 dataOut :
1314 dataOut :
1309 id :
1315 id :
1310 wintitle :
1316 wintitle :
1311 channelList :
1317 channelList :
1312 showProfile :
1318 showProfile :
1313 xmin : None,
1319 xmin : None,
1314 xmax : None,
1320 xmax : None,
1315 zmin : None,
1321 zmin : None,
1316 zmax : None
1322 zmax : None
1317 """
1323 """
1318
1324
1319 if cutHeight==None:
1325 if cutHeight==None:
1320 h=270
1326 h=270
1321 heightindex = numpy.abs(cutHeight - dataOut.heightList).argmin()
1327 heightindex = numpy.abs(cutHeight - dataOut.heightList).argmin()
1322 cutHeight = dataOut.heightList[heightindex]
1328 cutHeight = dataOut.heightList[heightindex]
1323
1329
1324 factor = dataOut.normFactor
1330 factor = dataOut.normFactor
1325 x = dataOut.abscissaList[:-1]
1331 x = dataOut.abscissaList[:-1]
1326 #y = dataOut.getHeiRange()
1332 #y = dataOut.getHeiRange()
1327
1333
1328 z = dataOut.data_pre[:,:,heightindex]/factor
1334 z = dataOut.data_pre[:,:,heightindex]/factor
1329 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
1335 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
1330 avg = numpy.average(z, axis=1)
1336 avg = numpy.average(z, axis=1)
1331 listChannels = z.shape[0]
1337 listChannels = z.shape[0]
1332
1338
1333 #Reconstruct Function
1339 #Reconstruct Function
1334 if fit==True:
1340 if fit==True:
1335 groupArray = dataOut.groupList
1341 groupArray = dataOut.groupList
1336 listChannels = groupArray.reshape((groupArray.size))
1342 listChannels = groupArray.reshape((groupArray.size))
1337 listChannels.sort()
1343 listChannels.sort()
1338 spcFitLine = numpy.zeros(z.shape)
1344 spcFitLine = numpy.zeros(z.shape)
1339 constants = dataOut.constants
1345 constants = dataOut.constants
1340
1346
1341 nGroups = groupArray.shape[0]
1347 nGroups = groupArray.shape[0]
1342 nChannels = groupArray.shape[1]
1348 nChannels = groupArray.shape[1]
1343 nProfiles = z.shape[1]
1349 nProfiles = z.shape[1]
1344
1350
1345 for f in range(nGroups):
1351 for f in range(nGroups):
1346 groupChann = groupArray[f,:]
1352 groupChann = groupArray[f,:]
1347 p = dataOut.data_param[f,:,heightindex]
1353 p = dataOut.data_param[f,:,heightindex]
1348 # p = numpy.array([ 89.343967,0.14036615,0.17086219,18.89835291,1.58388365,1.55099167])
1354 # p = numpy.array([ 89.343967,0.14036615,0.17086219,18.89835291,1.58388365,1.55099167])
1349 fitLineAux = dataOut.library.modelFunction(p, constants)*nProfiles
1355 fitLineAux = dataOut.library.modelFunction(p, constants)*nProfiles
1350 fitLineAux = fitLineAux.reshape((nChannels,nProfiles))
1356 fitLineAux = fitLineAux.reshape((nChannels,nProfiles))
1351 spcFitLine[groupChann,:] = fitLineAux
1357 spcFitLine[groupChann,:] = fitLineAux
1352 # spcFitLine = spcFitLine/factor
1358 # spcFitLine = spcFitLine/factor
1353
1359
1354 z = z[listChannels,:]
1360 z = z[listChannels,:]
1355 spcFitLine = spcFitLine[listChannels,:]
1361 spcFitLine = spcFitLine[listChannels,:]
1356 spcFitLinedB = 10*numpy.log10(spcFitLine)
1362 spcFitLinedB = 10*numpy.log10(spcFitLine)
1357
1363
1358 zdB = 10*numpy.log10(z)
1364 zdB = 10*numpy.log10(z)
1359 #thisDatetime = dataOut.datatime
1365 #thisDatetime = dataOut.datatime
1360 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1366 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1361 title = wintitle + " Doppler Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
1367 title = wintitle + " Doppler Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
1362 xlabel = "Velocity (m/s)"
1368 xlabel = "Velocity (m/s)"
1363 ylabel = "Spectrum"
1369 ylabel = "Spectrum"
1364
1370
1365 if not self.isConfig:
1371 if not self.isConfig:
1366
1372
1367 nplots = listChannels.size
1373 nplots = listChannels.size
1368
1374
1369 self.setup(id=id,
1375 self.setup(id=id,
1370 nplots=nplots,
1376 nplots=nplots,
1371 wintitle=wintitle,
1377 wintitle=wintitle,
1372 showprofile=showprofile,
1378 showprofile=showprofile,
1373 show=show)
1379 show=show)
1374
1380
1375 if xmin == None: xmin = numpy.nanmin(x)
1381 if xmin == None: xmin = numpy.nanmin(x)
1376 if xmax == None: xmax = numpy.nanmax(x)
1382 if xmax == None: xmax = numpy.nanmax(x)
1377 if ymin == None: ymin = numpy.nanmin(zdB)
1383 if ymin == None: ymin = numpy.nanmin(zdB)
1378 if ymax == None: ymax = numpy.nanmax(zdB)+2
1384 if ymax == None: ymax = numpy.nanmax(zdB)+2
1379
1385
1380 self.isConfig = True
1386 self.isConfig = True
1381
1387
1382 self.setWinTitle(title)
1388 self.setWinTitle(title)
1383 for i in range(self.nplots):
1389 for i in range(self.nplots):
1384 # title = "Channel %d: %4.2fdB" %(dataOut.channelList[i]+1, noisedB[i])
1390 # title = "Channel %d: %4.2fdB" %(dataOut.channelList[i]+1, noisedB[i])
1385 title = "Height %4.1f km\nChannel %d:" %(cutHeight, listChannels[i])
1391 title = "Height %4.1f km\nChannel %d:" %(cutHeight, listChannels[i])
1386 axes = self.axesList[i*self.__nsubplots]
1392 axes = self.axesList[i*self.__nsubplots]
1387 if fit == False:
1393 if fit == False:
1388 axes.pline(x, zdB[i,:],
1394 axes.pline(x, zdB[i,:],
1389 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1395 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1390 xlabel=xlabel, ylabel=ylabel, title=title
1396 xlabel=xlabel, ylabel=ylabel, title=title
1391 )
1397 )
1392 if fit == True:
1398 if fit == True:
1393 fitline=spcFitLinedB[i,:]
1399 fitline=spcFitLinedB[i,:]
1394 y=numpy.vstack([zdB[i,:],fitline] )
1400 y=numpy.vstack([zdB[i,:],fitline] )
1395 legendlabels=['Data','Fitting']
1401 legendlabels=['Data','Fitting']
1396 axes.pmultilineyaxis(x, y,
1402 axes.pmultilineyaxis(x, y,
1397 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1403 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1398 xlabel=xlabel, ylabel=ylabel, title=title,
1404 xlabel=xlabel, ylabel=ylabel, title=title,
1399 legendlabels=legendlabels, marker=None,
1405 legendlabels=legendlabels, marker=None,
1400 linestyle='solid', grid='both')
1406 linestyle='solid', grid='both')
1401
1407
1402 self.draw()
1408 self.draw()
1403
1409
1404 self.save(figpath=figpath,
1410 self.save(figpath=figpath,
1405 figfile=figfile,
1411 figfile=figfile,
1406 save=save,
1412 save=save,
1407 ftp=ftp,
1413 ftp=ftp,
1408 wr_period=wr_period,
1414 wr_period=wr_period,
1409 thisDatetime=thisDatetime)
1415 thisDatetime=thisDatetime)
1410
1416
1411
1417
1412 class EWDriftsPlot(Figure):
1418 class EWDriftsPlot(Figure):
1413
1419
1414 __isConfig = None
1420 __isConfig = None
1415 __nsubplots = None
1421 __nsubplots = None
1416
1422
1417 WIDTHPROF = None
1423 WIDTHPROF = None
1418 HEIGHTPROF = None
1424 HEIGHTPROF = None
1419 PREFIX = 'drift'
1425 PREFIX = 'drift'
1420
1426
1421 def __init__(self, **kwargs):
1427 def __init__(self, **kwargs):
1422 Figure.__init__(self, **kwargs)
1428 Figure.__init__(self, **kwargs)
1423 self.timerange = 2*60*60
1429 self.timerange = 2*60*60
1424 self.isConfig = False
1430 self.isConfig = False
1425 self.__nsubplots = 1
1431 self.__nsubplots = 1
1426
1432
1427 self.WIDTH = 800
1433 self.WIDTH = 800
1428 self.HEIGHT = 150
1434 self.HEIGHT = 150
1429 self.WIDTHPROF = 120
1435 self.WIDTHPROF = 120
1430 self.HEIGHTPROF = 0
1436 self.HEIGHTPROF = 0
1431 self.counter_imagwr = 0
1437 self.counter_imagwr = 0
1432
1438
1433 self.PLOT_CODE = EWDRIFT_CODE
1439 self.PLOT_CODE = EWDRIFT_CODE
1434
1440
1435 self.FTP_WEI = None
1441 self.FTP_WEI = None
1436 self.EXP_CODE = None
1442 self.EXP_CODE = None
1437 self.SUB_EXP_CODE = None
1443 self.SUB_EXP_CODE = None
1438 self.PLOT_POS = None
1444 self.PLOT_POS = None
1439 self.tmin = None
1445 self.tmin = None
1440 self.tmax = None
1446 self.tmax = None
1441
1447
1442 self.xmin = None
1448 self.xmin = None
1443 self.xmax = None
1449 self.xmax = None
1444
1450
1445 self.figfile = None
1451 self.figfile = None
1446
1452
1447 def getSubplots(self):
1453 def getSubplots(self):
1448
1454
1449 ncol = 1
1455 ncol = 1
1450 nrow = self.nplots
1456 nrow = self.nplots
1451
1457
1452 return nrow, ncol
1458 return nrow, ncol
1453
1459
1454 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1460 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1455
1461
1456 self.__showprofile = showprofile
1462 self.__showprofile = showprofile
1457 self.nplots = nplots
1463 self.nplots = nplots
1458
1464
1459 ncolspan = 1
1465 ncolspan = 1
1460 colspan = 1
1466 colspan = 1
1461
1467
1462 self.createFigure(id = id,
1468 self.createFigure(id = id,
1463 wintitle = wintitle,
1469 wintitle = wintitle,
1464 widthplot = self.WIDTH + self.WIDTHPROF,
1470 widthplot = self.WIDTH + self.WIDTHPROF,
1465 heightplot = self.HEIGHT + self.HEIGHTPROF,
1471 heightplot = self.HEIGHT + self.HEIGHTPROF,
1466 show=show)
1472 show=show)
1467
1473
1468 nrow, ncol = self.getSubplots()
1474 nrow, ncol = self.getSubplots()
1469
1475
1470 counter = 0
1476 counter = 0
1471 for y in range(nrow):
1477 for y in range(nrow):
1472 if counter >= self.nplots:
1478 if counter >= self.nplots:
1473 break
1479 break
1474
1480
1475 self.addAxes(nrow, ncol*ncolspan, y, 0, colspan, 1)
1481 self.addAxes(nrow, ncol*ncolspan, y, 0, colspan, 1)
1476 counter += 1
1482 counter += 1
1477
1483
1478 def run(self, dataOut, id, wintitle="", channelList=None,
1484 def run(self, dataOut, id, wintitle="", channelList=None,
1479 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
1485 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
1480 zmaxVertical = None, zminVertical = None, zmaxZonal = None, zminZonal = None,
1486 zmaxVertical = None, zminVertical = None, zmaxZonal = None, zminZonal = None,
1481 timerange=None, SNRthresh = -numpy.inf, SNRmin = None, SNRmax = None, SNR_1 = False,
1487 timerange=None, SNRthresh = -numpy.inf, SNRmin = None, SNRmax = None, SNR_1 = False,
1482 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
1488 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
1483 server=None, folder=None, username=None, password=None,
1489 server=None, folder=None, username=None, password=None,
1484 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1490 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1485 """
1491 """
1486
1492
1487 Input:
1493 Input:
1488 dataOut :
1494 dataOut :
1489 id :
1495 id :
1490 wintitle :
1496 wintitle :
1491 channelList :
1497 channelList :
1492 showProfile :
1498 showProfile :
1493 xmin : None,
1499 xmin : None,
1494 xmax : None,
1500 xmax : None,
1495 ymin : None,
1501 ymin : None,
1496 ymax : None,
1502 ymax : None,
1497 zmin : None,
1503 zmin : None,
1498 zmax : None
1504 zmax : None
1499 """
1505 """
1500
1506
1501 if timerange is not None:
1507 if timerange is not None:
1502 self.timerange = timerange
1508 self.timerange = timerange
1503
1509
1504 tmin = None
1510 tmin = None
1505 tmax = None
1511 tmax = None
1506
1512
1507 x = dataOut.getTimeRange1(dataOut.outputInterval)
1513 x = dataOut.getTimeRange1(dataOut.outputInterval)
1508 # y = dataOut.heightList
1514 # y = dataOut.heightList
1509 y = dataOut.heightList
1515 y = dataOut.heightList
1510
1516
1511 z = dataOut.data_output
1517 z = dataOut.data_output
1512 nplots = z.shape[0] #Number of wind dimensions estimated
1518 nplots = z.shape[0] #Number of wind dimensions estimated
1513 nplotsw = nplots
1519 nplotsw = nplots
1514
1520
1515 #If there is a SNR function defined
1521 #If there is a SNR function defined
1516 if dataOut.data_SNR is not None:
1522 if dataOut.data_SNR is not None:
1517 nplots += 1
1523 nplots += 1
1518 SNR = dataOut.data_SNR
1524 SNR = dataOut.data_SNR
1519
1525
1520 if SNR_1:
1526 if SNR_1:
1521 SNR += 1
1527 SNR += 1
1522
1528
1523 SNRavg = numpy.average(SNR, axis=0)
1529 SNRavg = numpy.average(SNR, axis=0)
1524
1530
1525 SNRdB = 10*numpy.log10(SNR)
1531 SNRdB = 10*numpy.log10(SNR)
1526 SNRavgdB = 10*numpy.log10(SNRavg)
1532 SNRavgdB = 10*numpy.log10(SNRavg)
1527
1533
1528 ind = numpy.where(SNRavg < 10**(SNRthresh/10))[0]
1534 ind = numpy.where(SNRavg < 10**(SNRthresh/10))[0]
1529
1535
1530 for i in range(nplotsw):
1536 for i in range(nplotsw):
1531 z[i,ind] = numpy.nan
1537 z[i,ind] = numpy.nan
1532
1538
1533
1539
1534 showprofile = False
1540 showprofile = False
1535 # thisDatetime = dataOut.datatime
1541 # thisDatetime = dataOut.datatime
1536 thisDatetime = datetime.datetime.utcfromtimestamp(x[1])
1542 thisDatetime = datetime.datetime.utcfromtimestamp(x[1])
1537 title = wintitle + " EW Drifts"
1543 title = wintitle + " EW Drifts"
1538 xlabel = ""
1544 xlabel = ""
1539 ylabel = "Height (Km)"
1545 ylabel = "Height (Km)"
1540
1546
1541 if not self.isConfig:
1547 if not self.isConfig:
1542
1548
1543 self.setup(id=id,
1549 self.setup(id=id,
1544 nplots=nplots,
1550 nplots=nplots,
1545 wintitle=wintitle,
1551 wintitle=wintitle,
1546 showprofile=showprofile,
1552 showprofile=showprofile,
1547 show=show)
1553 show=show)
1548
1554
1549 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1555 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1550
1556
1551 if ymin == None: ymin = numpy.nanmin(y)
1557 if ymin == None: ymin = numpy.nanmin(y)
1552 if ymax == None: ymax = numpy.nanmax(y)
1558 if ymax == None: ymax = numpy.nanmax(y)
1553
1559
1554 if zmaxZonal == None: zmaxZonal = numpy.nanmax(abs(z[0,:]))
1560 if zmaxZonal == None: zmaxZonal = numpy.nanmax(abs(z[0,:]))
1555 if zminZonal == None: zminZonal = -zmaxZonal
1561 if zminZonal == None: zminZonal = -zmaxZonal
1556 if zmaxVertical == None: zmaxVertical = numpy.nanmax(abs(z[1,:]))
1562 if zmaxVertical == None: zmaxVertical = numpy.nanmax(abs(z[1,:]))
1557 if zminVertical == None: zminVertical = -zmaxVertical
1563 if zminVertical == None: zminVertical = -zmaxVertical
1558
1564
1559 if dataOut.data_SNR is not None:
1565 if dataOut.data_SNR is not None:
1560 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
1566 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
1561 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
1567 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
1562
1568
1563 self.FTP_WEI = ftp_wei
1569 self.FTP_WEI = ftp_wei
1564 self.EXP_CODE = exp_code
1570 self.EXP_CODE = exp_code
1565 self.SUB_EXP_CODE = sub_exp_code
1571 self.SUB_EXP_CODE = sub_exp_code
1566 self.PLOT_POS = plot_pos
1572 self.PLOT_POS = plot_pos
1567
1573
1568 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1574 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1569 self.isConfig = True
1575 self.isConfig = True
1570
1576
1571
1577
1572 self.setWinTitle(title)
1578 self.setWinTitle(title)
1573
1579
1574 if ((self.xmax - x[1]) < (x[1]-x[0])):
1580 if ((self.xmax - x[1]) < (x[1]-x[0])):
1575 x[1] = self.xmax
1581 x[1] = self.xmax
1576
1582
1577 strWind = ['Zonal','Vertical']
1583 strWind = ['Zonal','Vertical']
1578 strCb = 'Velocity (m/s)'
1584 strCb = 'Velocity (m/s)'
1579 zmaxVector = [zmaxZonal, zmaxVertical]
1585 zmaxVector = [zmaxZonal, zmaxVertical]
1580 zminVector = [zminZonal, zminVertical]
1586 zminVector = [zminZonal, zminVertical]
1581
1587
1582 for i in range(nplotsw):
1588 for i in range(nplotsw):
1583
1589
1584 title = "%s Drifts: %s" %(strWind[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1590 title = "%s Drifts: %s" %(strWind[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1585 axes = self.axesList[i*self.__nsubplots]
1591 axes = self.axesList[i*self.__nsubplots]
1586
1592
1587 z1 = z[i,:].reshape((1,-1))
1593 z1 = z[i,:].reshape((1,-1))
1588
1594
1589 axes.pcolorbuffer(x, y, z1,
1595 axes.pcolorbuffer(x, y, z1,
1590 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zminVector[i], zmax=zmaxVector[i],
1596 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zminVector[i], zmax=zmaxVector[i],
1591 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
1597 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
1592 ticksize=9, cblabel=strCb, cbsize="1%", colormap="RdBu_r")
1598 ticksize=9, cblabel=strCb, cbsize="1%", colormap="RdBu_r")
1593
1599
1594 if dataOut.data_SNR is not None:
1600 if dataOut.data_SNR is not None:
1595 i += 1
1601 i += 1
1596 if SNR_1:
1602 if SNR_1:
1597 title = "Signal Noise Ratio + 1 (SNR+1): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1603 title = "Signal Noise Ratio + 1 (SNR+1): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1598 else:
1604 else:
1599 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1605 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1600 axes = self.axesList[i*self.__nsubplots]
1606 axes = self.axesList[i*self.__nsubplots]
1601 SNRavgdB = SNRavgdB.reshape((1,-1))
1607 SNRavgdB = SNRavgdB.reshape((1,-1))
1602
1608
1603 axes.pcolorbuffer(x, y, SNRavgdB,
1609 axes.pcolorbuffer(x, y, SNRavgdB,
1604 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
1610 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
1605 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
1611 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
1606 ticksize=9, cblabel='', cbsize="1%", colormap="jet")
1612 ticksize=9, cblabel='', cbsize="1%", colormap="jet")
1607
1613
1608 self.draw()
1614 self.draw()
1609
1615
1610 if x[1] >= self.axesList[0].xmax:
1616 if x[1] >= self.axesList[0].xmax:
1611 self.counter_imagwr = wr_period
1617 self.counter_imagwr = wr_period
1612 self.isConfig = False
1618 self.isConfig = False
1613 self.figfile = None
1619 self.figfile = None
1614
1620
1615
1621
1616
1622
1617
1623
1618 class PhasePlot(Figure):
1624 class PhasePlot(Figure):
1619
1625
1620 __isConfig = None
1626 __isConfig = None
1621 __nsubplots = None
1627 __nsubplots = None
1622
1628
1623 PREFIX = 'mphase'
1629 PREFIX = 'mphase'
1624
1630
1625
1631
1626 def __init__(self, **kwargs):
1632 def __init__(self, **kwargs):
1627 Figure.__init__(self, **kwargs)
1633 Figure.__init__(self, **kwargs)
1628 self.timerange = 24*60*60
1634 self.timerange = 24*60*60
1629 self.isConfig = False
1635 self.isConfig = False
1630 self.__nsubplots = 1
1636 self.__nsubplots = 1
1631 self.counter_imagwr = 0
1637 self.counter_imagwr = 0
1632 self.WIDTH = 600
1638 self.WIDTH = 600
1633 self.HEIGHT = 300
1639 self.HEIGHT = 300
1634 self.WIDTHPROF = 120
1640 self.WIDTHPROF = 120
1635 self.HEIGHTPROF = 0
1641 self.HEIGHTPROF = 0
1636 self.xdata = None
1642 self.xdata = None
1637 self.ydata = None
1643 self.ydata = None
1638
1644
1639 self.PLOT_CODE = MPHASE_CODE
1645 self.PLOT_CODE = MPHASE_CODE
1640
1646
1641 self.FTP_WEI = None
1647 self.FTP_WEI = None
1642 self.EXP_CODE = None
1648 self.EXP_CODE = None
1643 self.SUB_EXP_CODE = None
1649 self.SUB_EXP_CODE = None
1644 self.PLOT_POS = None
1650 self.PLOT_POS = None
1645
1651
1646
1652
1647 self.filename_phase = None
1653 self.filename_phase = None
1648
1654
1649 self.figfile = None
1655 self.figfile = None
1650
1656
1651 def getSubplots(self):
1657 def getSubplots(self):
1652
1658
1653 ncol = 1
1659 ncol = 1
1654 nrow = 1
1660 nrow = 1
1655
1661
1656 return nrow, ncol
1662 return nrow, ncol
1657
1663
1658 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1664 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1659
1665
1660 self.__showprofile = showprofile
1666 self.__showprofile = showprofile
1661 self.nplots = nplots
1667 self.nplots = nplots
1662
1668
1663 ncolspan = 7
1669 ncolspan = 7
1664 colspan = 6
1670 colspan = 6
1665 self.__nsubplots = 2
1671 self.__nsubplots = 2
1666
1672
1667 self.createFigure(id = id,
1673 self.createFigure(id = id,
1668 wintitle = wintitle,
1674 wintitle = wintitle,
1669 widthplot = self.WIDTH+self.WIDTHPROF,
1675 widthplot = self.WIDTH+self.WIDTHPROF,
1670 heightplot = self.HEIGHT+self.HEIGHTPROF,
1676 heightplot = self.HEIGHT+self.HEIGHTPROF,
1671 show=show)
1677 show=show)
1672
1678
1673 nrow, ncol = self.getSubplots()
1679 nrow, ncol = self.getSubplots()
1674
1680
1675 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1681 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1676
1682
1677
1683
1678 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
1684 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
1679 xmin=None, xmax=None, ymin=None, ymax=None,
1685 xmin=None, xmax=None, ymin=None, ymax=None,
1680 timerange=None,
1686 timerange=None,
1681 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1687 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1682 server=None, folder=None, username=None, password=None,
1688 server=None, folder=None, username=None, password=None,
1683 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1689 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1684
1690
1685
1691
1686 tmin = None
1692 tmin = None
1687 tmax = None
1693 tmax = None
1688 x = dataOut.getTimeRange1(dataOut.outputInterval)
1694 x = dataOut.getTimeRange1(dataOut.outputInterval)
1689 y = dataOut.getHeiRange()
1695 y = dataOut.getHeiRange()
1690
1696
1691
1697
1692 #thisDatetime = dataOut.datatime
1698 #thisDatetime = dataOut.datatime
1693 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
1699 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
1694 title = wintitle + " Phase of Beacon Signal" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1700 title = wintitle + " Phase of Beacon Signal" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1695 xlabel = "Local Time"
1701 xlabel = "Local Time"
1696 ylabel = "Phase"
1702 ylabel = "Phase"
1697
1703
1698
1704
1699 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
1705 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
1700 phase_beacon = dataOut.data_output
1706 phase_beacon = dataOut.data_output
1701 update_figfile = False
1707 update_figfile = False
1702
1708
1703 if not self.isConfig:
1709 if not self.isConfig:
1704
1710
1705 self.nplots = phase_beacon.size
1711 self.nplots = phase_beacon.size
1706
1712
1707 self.setup(id=id,
1713 self.setup(id=id,
1708 nplots=self.nplots,
1714 nplots=self.nplots,
1709 wintitle=wintitle,
1715 wintitle=wintitle,
1710 showprofile=showprofile,
1716 showprofile=showprofile,
1711 show=show)
1717 show=show)
1712
1718
1713 if timerange is not None:
1719 if timerange is not None:
1714 self.timerange = timerange
1720 self.timerange = timerange
1715
1721
1716 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1722 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1717
1723
1718 if ymin == None: ymin = numpy.nanmin(phase_beacon) - 10.0
1724 if ymin == None: ymin = numpy.nanmin(phase_beacon) - 10.0
1719 if ymax == None: ymax = numpy.nanmax(phase_beacon) + 10.0
1725 if ymax == None: ymax = numpy.nanmax(phase_beacon) + 10.0
1720
1726
1721 self.FTP_WEI = ftp_wei
1727 self.FTP_WEI = ftp_wei
1722 self.EXP_CODE = exp_code
1728 self.EXP_CODE = exp_code
1723 self.SUB_EXP_CODE = sub_exp_code
1729 self.SUB_EXP_CODE = sub_exp_code
1724 self.PLOT_POS = plot_pos
1730 self.PLOT_POS = plot_pos
1725
1731
1726 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1732 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1727 self.isConfig = True
1733 self.isConfig = True
1728 self.figfile = figfile
1734 self.figfile = figfile
1729 self.xdata = numpy.array([])
1735 self.xdata = numpy.array([])
1730 self.ydata = numpy.array([])
1736 self.ydata = numpy.array([])
1731
1737
1732 #open file beacon phase
1738 #open file beacon phase
1733 path = '%s%03d' %(self.PREFIX, self.id)
1739 path = '%s%03d' %(self.PREFIX, self.id)
1734 beacon_file = os.path.join(path,'%s.txt'%self.name)
1740 beacon_file = os.path.join(path,'%s.txt'%self.name)
1735 self.filename_phase = os.path.join(figpath,beacon_file)
1741 self.filename_phase = os.path.join(figpath,beacon_file)
1736 update_figfile = True
1742 update_figfile = True
1737
1743
1738
1744
1739 #store data beacon phase
1745 #store data beacon phase
1740 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
1746 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
1741
1747
1742 self.setWinTitle(title)
1748 self.setWinTitle(title)
1743
1749
1744
1750
1745 title = "Phase Offset %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1751 title = "Phase Offset %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1746
1752
1747 legendlabels = ["phase %d"%(chan) for chan in numpy.arange(self.nplots)]
1753 legendlabels = ["phase %d"%(chan) for chan in numpy.arange(self.nplots)]
1748
1754
1749 axes = self.axesList[0]
1755 axes = self.axesList[0]
1750
1756
1751 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1757 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1752
1758
1753 if len(self.ydata)==0:
1759 if len(self.ydata)==0:
1754 self.ydata = phase_beacon.reshape(-1,1)
1760 self.ydata = phase_beacon.reshape(-1,1)
1755 else:
1761 else:
1756 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
1762 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
1757
1763
1758
1764
1759 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1765 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1760 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1766 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1761 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1767 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1762 XAxisAsTime=True, grid='both'
1768 XAxisAsTime=True, grid='both'
1763 )
1769 )
1764
1770
1765 self.draw()
1771 self.draw()
1766
1772
1767 self.save(figpath=figpath,
1773 self.save(figpath=figpath,
1768 figfile=figfile,
1774 figfile=figfile,
1769 save=save,
1775 save=save,
1770 ftp=ftp,
1776 ftp=ftp,
1771 wr_period=wr_period,
1777 wr_period=wr_period,
1772 thisDatetime=thisDatetime,
1778 thisDatetime=thisDatetime,
1773 update_figfile=update_figfile)
1779 update_figfile=update_figfile)
1774
1780
1775 if dataOut.ltctime + dataOut.outputInterval >= self.xmax:
1781 if dataOut.ltctime + dataOut.outputInterval >= self.xmax:
1776 self.counter_imagwr = wr_period
1782 self.counter_imagwr = wr_period
1777 self.isConfig = False
1783 self.isConfig = False
1778 update_figfile = True
1784 update_figfile = True
1779
1785
1780
1786
1781
1787
1782 class NSMeteorDetection1Plot(Figure):
1788 class NSMeteorDetection1Plot(Figure):
1783
1789
1784 isConfig = None
1790 isConfig = None
1785 __nsubplots = None
1791 __nsubplots = None
1786
1792
1787 WIDTHPROF = None
1793 WIDTHPROF = None
1788 HEIGHTPROF = None
1794 HEIGHTPROF = None
1789 PREFIX = 'nsm'
1795 PREFIX = 'nsm'
1790
1796
1791 zminList = None
1797 zminList = None
1792 zmaxList = None
1798 zmaxList = None
1793 cmapList = None
1799 cmapList = None
1794 titleList = None
1800 titleList = None
1795 nPairs = None
1801 nPairs = None
1796 nChannels = None
1802 nChannels = None
1797 nParam = None
1803 nParam = None
1798
1804
1799 def __init__(self, **kwargs):
1805 def __init__(self, **kwargs):
1800 Figure.__init__(self, **kwargs)
1806 Figure.__init__(self, **kwargs)
1801 self.isConfig = False
1807 self.isConfig = False
1802 self.__nsubplots = 1
1808 self.__nsubplots = 1
1803
1809
1804 self.WIDTH = 750
1810 self.WIDTH = 750
1805 self.HEIGHT = 250
1811 self.HEIGHT = 250
1806 self.WIDTHPROF = 120
1812 self.WIDTHPROF = 120
1807 self.HEIGHTPROF = 0
1813 self.HEIGHTPROF = 0
1808 self.counter_imagwr = 0
1814 self.counter_imagwr = 0
1809
1815
1810 self.PLOT_CODE = SPEC_CODE
1816 self.PLOT_CODE = SPEC_CODE
1811
1817
1812 self.FTP_WEI = None
1818 self.FTP_WEI = None
1813 self.EXP_CODE = None
1819 self.EXP_CODE = None
1814 self.SUB_EXP_CODE = None
1820 self.SUB_EXP_CODE = None
1815 self.PLOT_POS = None
1821 self.PLOT_POS = None
1816
1822
1817 self.__xfilter_ena = False
1823 self.__xfilter_ena = False
1818 self.__yfilter_ena = False
1824 self.__yfilter_ena = False
1819
1825
1820 def getSubplots(self):
1826 def getSubplots(self):
1821
1827
1822 ncol = 3
1828 ncol = 3
1823 nrow = int(numpy.ceil(self.nplots/3.0))
1829 nrow = int(numpy.ceil(self.nplots/3.0))
1824
1830
1825 return nrow, ncol
1831 return nrow, ncol
1826
1832
1827 def setup(self, id, nplots, wintitle, show=True):
1833 def setup(self, id, nplots, wintitle, show=True):
1828
1834
1829 self.nplots = nplots
1835 self.nplots = nplots
1830
1836
1831 ncolspan = 1
1837 ncolspan = 1
1832 colspan = 1
1838 colspan = 1
1833
1839
1834 self.createFigure(id = id,
1840 self.createFigure(id = id,
1835 wintitle = wintitle,
1841 wintitle = wintitle,
1836 widthplot = self.WIDTH + self.WIDTHPROF,
1842 widthplot = self.WIDTH + self.WIDTHPROF,
1837 heightplot = self.HEIGHT + self.HEIGHTPROF,
1843 heightplot = self.HEIGHT + self.HEIGHTPROF,
1838 show=show)
1844 show=show)
1839
1845
1840 nrow, ncol = self.getSubplots()
1846 nrow, ncol = self.getSubplots()
1841
1847
1842 counter = 0
1848 counter = 0
1843 for y in range(nrow):
1849 for y in range(nrow):
1844 for x in range(ncol):
1850 for x in range(ncol):
1845
1851
1846 if counter >= self.nplots:
1852 if counter >= self.nplots:
1847 break
1853 break
1848
1854
1849 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1855 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1850
1856
1851 counter += 1
1857 counter += 1
1852
1858
1853 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
1859 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
1854 xmin=None, xmax=None, ymin=None, ymax=None, SNRmin=None, SNRmax=None,
1860 xmin=None, xmax=None, ymin=None, ymax=None, SNRmin=None, SNRmax=None,
1855 vmin=None, vmax=None, wmin=None, wmax=None, mode = 'SA',
1861 vmin=None, vmax=None, wmin=None, wmax=None, mode = 'SA',
1856 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1862 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1857 server=None, folder=None, username=None, password=None,
1863 server=None, folder=None, username=None, password=None,
1858 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
1864 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
1859 xaxis="frequency"):
1865 xaxis="frequency"):
1860
1866
1861 """
1867 """
1862
1868
1863 Input:
1869 Input:
1864 dataOut :
1870 dataOut :
1865 id :
1871 id :
1866 wintitle :
1872 wintitle :
1867 channelList :
1873 channelList :
1868 showProfile :
1874 showProfile :
1869 xmin : None,
1875 xmin : None,
1870 xmax : None,
1876 xmax : None,
1871 ymin : None,
1877 ymin : None,
1872 ymax : None,
1878 ymax : None,
1873 zmin : None,
1879 zmin : None,
1874 zmax : None
1880 zmax : None
1875 """
1881 """
1876 #SEPARAR EN DOS PLOTS
1882 #SEPARAR EN DOS PLOTS
1877 nParam = dataOut.data_param.shape[1] - 3
1883 nParam = dataOut.data_param.shape[1] - 3
1878
1884
1879 utctime = dataOut.data_param[0,0]
1885 utctime = dataOut.data_param[0,0]
1880 tmet = dataOut.data_param[:,1].astype(int)
1886 tmet = dataOut.data_param[:,1].astype(int)
1881 hmet = dataOut.data_param[:,2].astype(int)
1887 hmet = dataOut.data_param[:,2].astype(int)
1882
1888
1883 x = dataOut.abscissaList
1889 x = dataOut.abscissaList
1884 y = dataOut.heightList
1890 y = dataOut.heightList
1885
1891
1886 z = numpy.zeros((nParam, y.size, x.size - 1))
1892 z = numpy.zeros((nParam, y.size, x.size - 1))
1887 z[:,:] = numpy.nan
1893 z[:,:] = numpy.nan
1888 z[:,hmet,tmet] = dataOut.data_param[:,3:].T
1894 z[:,hmet,tmet] = dataOut.data_param[:,3:].T
1889 z[0,:,:] = 10*numpy.log10(z[0,:,:])
1895 z[0,:,:] = 10*numpy.log10(z[0,:,:])
1890
1896
1891 xlabel = "Time (s)"
1897 xlabel = "Time (s)"
1892 ylabel = "Range (km)"
1898 ylabel = "Range (km)"
1893
1899
1894 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
1900 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
1895
1901
1896 if not self.isConfig:
1902 if not self.isConfig:
1897
1903
1898 nplots = nParam
1904 nplots = nParam
1899
1905
1900 self.setup(id=id,
1906 self.setup(id=id,
1901 nplots=nplots,
1907 nplots=nplots,
1902 wintitle=wintitle,
1908 wintitle=wintitle,
1903 show=show)
1909 show=show)
1904
1910
1905 if xmin is None: xmin = numpy.nanmin(x)
1911 if xmin is None: xmin = numpy.nanmin(x)
1906 if xmax is None: xmax = numpy.nanmax(x)
1912 if xmax is None: xmax = numpy.nanmax(x)
1907 if ymin is None: ymin = numpy.nanmin(y)
1913 if ymin is None: ymin = numpy.nanmin(y)
1908 if ymax is None: ymax = numpy.nanmax(y)
1914 if ymax is None: ymax = numpy.nanmax(y)
1909 if SNRmin is None: SNRmin = numpy.nanmin(z[0,:])
1915 if SNRmin is None: SNRmin = numpy.nanmin(z[0,:])
1910 if SNRmax is None: SNRmax = numpy.nanmax(z[0,:])
1916 if SNRmax is None: SNRmax = numpy.nanmax(z[0,:])
1911 if vmax is None: vmax = numpy.nanmax(numpy.abs(z[1,:]))
1917 if vmax is None: vmax = numpy.nanmax(numpy.abs(z[1,:]))
1912 if vmin is None: vmin = -vmax
1918 if vmin is None: vmin = -vmax
1913 if wmin is None: wmin = 0
1919 if wmin is None: wmin = 0
1914 if wmax is None: wmax = 50
1920 if wmax is None: wmax = 50
1915
1921
1916 pairsList = dataOut.groupList
1922 pairsList = dataOut.groupList
1917 self.nPairs = len(dataOut.groupList)
1923 self.nPairs = len(dataOut.groupList)
1918
1924
1919 zminList = [SNRmin, vmin, cmin] + [pmin]*self.nPairs
1925 zminList = [SNRmin, vmin, cmin] + [pmin]*self.nPairs
1920 zmaxList = [SNRmax, vmax, cmax] + [pmax]*self.nPairs
1926 zmaxList = [SNRmax, vmax, cmax] + [pmax]*self.nPairs
1921 titleList = ["SNR","Radial Velocity","Coherence"]
1927 titleList = ["SNR","Radial Velocity","Coherence"]
1922 cmapList = ["jet","RdBu_r","jet"]
1928 cmapList = ["jet","RdBu_r","jet"]
1923
1929
1924 for i in range(self.nPairs):
1930 for i in range(self.nPairs):
1925 strAux1 = "Phase Difference "+ str(pairsList[i][0]) + str(pairsList[i][1])
1931 strAux1 = "Phase Difference "+ str(pairsList[i][0]) + str(pairsList[i][1])
1926 titleList = titleList + [strAux1]
1932 titleList = titleList + [strAux1]
1927 cmapList = cmapList + ["RdBu_r"]
1933 cmapList = cmapList + ["RdBu_r"]
1928
1934
1929 self.zminList = zminList
1935 self.zminList = zminList
1930 self.zmaxList = zmaxList
1936 self.zmaxList = zmaxList
1931 self.cmapList = cmapList
1937 self.cmapList = cmapList
1932 self.titleList = titleList
1938 self.titleList = titleList
1933
1939
1934 self.FTP_WEI = ftp_wei
1940 self.FTP_WEI = ftp_wei
1935 self.EXP_CODE = exp_code
1941 self.EXP_CODE = exp_code
1936 self.SUB_EXP_CODE = sub_exp_code
1942 self.SUB_EXP_CODE = sub_exp_code
1937 self.PLOT_POS = plot_pos
1943 self.PLOT_POS = plot_pos
1938
1944
1939 self.isConfig = True
1945 self.isConfig = True
1940
1946
1941 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
1947 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
1942
1948
1943 for i in range(nParam):
1949 for i in range(nParam):
1944 title = self.titleList[i] + ": " +str_datetime
1950 title = self.titleList[i] + ": " +str_datetime
1945 axes = self.axesList[i]
1951 axes = self.axesList[i]
1946 axes.pcolor(x, y, z[i,:].T,
1952 axes.pcolor(x, y, z[i,:].T,
1947 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=self.zminList[i], zmax=self.zmaxList[i],
1953 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=self.zminList[i], zmax=self.zmaxList[i],
1948 xlabel=xlabel, ylabel=ylabel, title=title, colormap=self.cmapList[i],ticksize=9, cblabel='')
1954 xlabel=xlabel, ylabel=ylabel, title=title, colormap=self.cmapList[i],ticksize=9, cblabel='')
1949 self.draw()
1955 self.draw()
1950
1956
1951 if figfile == None:
1957 if figfile == None:
1952 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
1958 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
1953 name = str_datetime
1959 name = str_datetime
1954 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
1960 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
1955 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
1961 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
1956 figfile = self.getFilename(name)
1962 figfile = self.getFilename(name)
1957
1963
1958 self.save(figpath=figpath,
1964 self.save(figpath=figpath,
1959 figfile=figfile,
1965 figfile=figfile,
1960 save=save,
1966 save=save,
1961 ftp=ftp,
1967 ftp=ftp,
1962 wr_period=wr_period,
1968 wr_period=wr_period,
1963 thisDatetime=thisDatetime)
1969 thisDatetime=thisDatetime)
1964
1970
1965
1971
1966 class NSMeteorDetection2Plot(Figure):
1972 class NSMeteorDetection2Plot(Figure):
1967
1973
1968 isConfig = None
1974 isConfig = None
1969 __nsubplots = None
1975 __nsubplots = None
1970
1976
1971 WIDTHPROF = None
1977 WIDTHPROF = None
1972 HEIGHTPROF = None
1978 HEIGHTPROF = None
1973 PREFIX = 'nsm'
1979 PREFIX = 'nsm'
1974
1980
1975 zminList = None
1981 zminList = None
1976 zmaxList = None
1982 zmaxList = None
1977 cmapList = None
1983 cmapList = None
1978 titleList = None
1984 titleList = None
1979 nPairs = None
1985 nPairs = None
1980 nChannels = None
1986 nChannels = None
1981 nParam = None
1987 nParam = None
1982
1988
1983 def __init__(self, **kwargs):
1989 def __init__(self, **kwargs):
1984 Figure.__init__(self, **kwargs)
1990 Figure.__init__(self, **kwargs)
1985 self.isConfig = False
1991 self.isConfig = False
1986 self.__nsubplots = 1
1992 self.__nsubplots = 1
1987
1993
1988 self.WIDTH = 750
1994 self.WIDTH = 750
1989 self.HEIGHT = 250
1995 self.HEIGHT = 250
1990 self.WIDTHPROF = 120
1996 self.WIDTHPROF = 120
1991 self.HEIGHTPROF = 0
1997 self.HEIGHTPROF = 0
1992 self.counter_imagwr = 0
1998 self.counter_imagwr = 0
1993
1999
1994 self.PLOT_CODE = SPEC_CODE
2000 self.PLOT_CODE = SPEC_CODE
1995
2001
1996 self.FTP_WEI = None
2002 self.FTP_WEI = None
1997 self.EXP_CODE = None
2003 self.EXP_CODE = None
1998 self.SUB_EXP_CODE = None
2004 self.SUB_EXP_CODE = None
1999 self.PLOT_POS = None
2005 self.PLOT_POS = None
2000
2006
2001 self.__xfilter_ena = False
2007 self.__xfilter_ena = False
2002 self.__yfilter_ena = False
2008 self.__yfilter_ena = False
2003
2009
2004 def getSubplots(self):
2010 def getSubplots(self):
2005
2011
2006 ncol = 3
2012 ncol = 3
2007 nrow = int(numpy.ceil(self.nplots/3.0))
2013 nrow = int(numpy.ceil(self.nplots/3.0))
2008
2014
2009 return nrow, ncol
2015 return nrow, ncol
2010
2016
2011 def setup(self, id, nplots, wintitle, show=True):
2017 def setup(self, id, nplots, wintitle, show=True):
2012
2018
2013 self.nplots = nplots
2019 self.nplots = nplots
2014
2020
2015 ncolspan = 1
2021 ncolspan = 1
2016 colspan = 1
2022 colspan = 1
2017
2023
2018 self.createFigure(id = id,
2024 self.createFigure(id = id,
2019 wintitle = wintitle,
2025 wintitle = wintitle,
2020 widthplot = self.WIDTH + self.WIDTHPROF,
2026 widthplot = self.WIDTH + self.WIDTHPROF,
2021 heightplot = self.HEIGHT + self.HEIGHTPROF,
2027 heightplot = self.HEIGHT + self.HEIGHTPROF,
2022 show=show)
2028 show=show)
2023
2029
2024 nrow, ncol = self.getSubplots()
2030 nrow, ncol = self.getSubplots()
2025
2031
2026 counter = 0
2032 counter = 0
2027 for y in range(nrow):
2033 for y in range(nrow):
2028 for x in range(ncol):
2034 for x in range(ncol):
2029
2035
2030 if counter >= self.nplots:
2036 if counter >= self.nplots:
2031 break
2037 break
2032
2038
2033 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
2039 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
2034
2040
2035 counter += 1
2041 counter += 1
2036
2042
2037 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
2043 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
2038 xmin=None, xmax=None, ymin=None, ymax=None, SNRmin=None, SNRmax=None,
2044 xmin=None, xmax=None, ymin=None, ymax=None, SNRmin=None, SNRmax=None,
2039 vmin=None, vmax=None, wmin=None, wmax=None, mode = 'SA',
2045 vmin=None, vmax=None, wmin=None, wmax=None, mode = 'SA',
2040 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
2046 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
2041 server=None, folder=None, username=None, password=None,
2047 server=None, folder=None, username=None, password=None,
2042 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
2048 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
2043 xaxis="frequency"):
2049 xaxis="frequency"):
2044
2050
2045 """
2051 """
2046
2052
2047 Input:
2053 Input:
2048 dataOut :
2054 dataOut :
2049 id :
2055 id :
2050 wintitle :
2056 wintitle :
2051 channelList :
2057 channelList :
2052 showProfile :
2058 showProfile :
2053 xmin : None,
2059 xmin : None,
2054 xmax : None,
2060 xmax : None,
2055 ymin : None,
2061 ymin : None,
2056 ymax : None,
2062 ymax : None,
2057 zmin : None,
2063 zmin : None,
2058 zmax : None
2064 zmax : None
2059 """
2065 """
2060 #Rebuild matrix
2066 #Rebuild matrix
2061 utctime = dataOut.data_param[0,0]
2067 utctime = dataOut.data_param[0,0]
2062 cmet = dataOut.data_param[:,1].astype(int)
2068 cmet = dataOut.data_param[:,1].astype(int)
2063 tmet = dataOut.data_param[:,2].astype(int)
2069 tmet = dataOut.data_param[:,2].astype(int)
2064 hmet = dataOut.data_param[:,3].astype(int)
2070 hmet = dataOut.data_param[:,3].astype(int)
2065
2071
2066 nParam = 3
2072 nParam = 3
2067 nChan = len(dataOut.groupList)
2073 nChan = len(dataOut.groupList)
2068 x = dataOut.abscissaList
2074 x = dataOut.abscissaList
2069 y = dataOut.heightList
2075 y = dataOut.heightList
2070
2076
2071 z = numpy.full((nChan, nParam, y.size, x.size - 1),numpy.nan)
2077 z = numpy.full((nChan, nParam, y.size, x.size - 1),numpy.nan)
2072 z[cmet,:,hmet,tmet] = dataOut.data_param[:,4:]
2078 z[cmet,:,hmet,tmet] = dataOut.data_param[:,4:]
2073 z[:,0,:,:] = 10*numpy.log10(z[:,0,:,:]) #logarithmic scale
2079 z[:,0,:,:] = 10*numpy.log10(z[:,0,:,:]) #logarithmic scale
2074 z = numpy.reshape(z, (nChan*nParam, y.size, x.size-1))
2080 z = numpy.reshape(z, (nChan*nParam, y.size, x.size-1))
2075
2081
2076 xlabel = "Time (s)"
2082 xlabel = "Time (s)"
2077 ylabel = "Range (km)"
2083 ylabel = "Range (km)"
2078
2084
2079 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
2085 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.ltctime)
2080
2086
2081 if not self.isConfig:
2087 if not self.isConfig:
2082
2088
2083 nplots = nParam*nChan
2089 nplots = nParam*nChan
2084
2090
2085 self.setup(id=id,
2091 self.setup(id=id,
2086 nplots=nplots,
2092 nplots=nplots,
2087 wintitle=wintitle,
2093 wintitle=wintitle,
2088 show=show)
2094 show=show)
2089
2095
2090 if xmin is None: xmin = numpy.nanmin(x)
2096 if xmin is None: xmin = numpy.nanmin(x)
2091 if xmax is None: xmax = numpy.nanmax(x)
2097 if xmax is None: xmax = numpy.nanmax(x)
2092 if ymin is None: ymin = numpy.nanmin(y)
2098 if ymin is None: ymin = numpy.nanmin(y)
2093 if ymax is None: ymax = numpy.nanmax(y)
2099 if ymax is None: ymax = numpy.nanmax(y)
2094 if SNRmin is None: SNRmin = numpy.nanmin(z[0,:])
2100 if SNRmin is None: SNRmin = numpy.nanmin(z[0,:])
2095 if SNRmax is None: SNRmax = numpy.nanmax(z[0,:])
2101 if SNRmax is None: SNRmax = numpy.nanmax(z[0,:])
2096 if vmax is None: vmax = numpy.nanmax(numpy.abs(z[1,:]))
2102 if vmax is None: vmax = numpy.nanmax(numpy.abs(z[1,:]))
2097 if vmin is None: vmin = -vmax
2103 if vmin is None: vmin = -vmax
2098 if wmin is None: wmin = 0
2104 if wmin is None: wmin = 0
2099 if wmax is None: wmax = 50
2105 if wmax is None: wmax = 50
2100
2106
2101 self.nChannels = nChan
2107 self.nChannels = nChan
2102
2108
2103 zminList = []
2109 zminList = []
2104 zmaxList = []
2110 zmaxList = []
2105 titleList = []
2111 titleList = []
2106 cmapList = []
2112 cmapList = []
2107 for i in range(self.nChannels):
2113 for i in range(self.nChannels):
2108 strAux1 = "SNR Channel "+ str(i)
2114 strAux1 = "SNR Channel "+ str(i)
2109 strAux2 = "Radial Velocity Channel "+ str(i)
2115 strAux2 = "Radial Velocity Channel "+ str(i)
2110 strAux3 = "Spectral Width Channel "+ str(i)
2116 strAux3 = "Spectral Width Channel "+ str(i)
2111
2117
2112 titleList = titleList + [strAux1,strAux2,strAux3]
2118 titleList = titleList + [strAux1,strAux2,strAux3]
2113 cmapList = cmapList + ["jet","RdBu_r","jet"]
2119 cmapList = cmapList + ["jet","RdBu_r","jet"]
2114 zminList = zminList + [SNRmin,vmin,wmin]
2120 zminList = zminList + [SNRmin,vmin,wmin]
2115 zmaxList = zmaxList + [SNRmax,vmax,wmax]
2121 zmaxList = zmaxList + [SNRmax,vmax,wmax]
2116
2122
2117 self.zminList = zminList
2123 self.zminList = zminList
2118 self.zmaxList = zmaxList
2124 self.zmaxList = zmaxList
2119 self.cmapList = cmapList
2125 self.cmapList = cmapList
2120 self.titleList = titleList
2126 self.titleList = titleList
2121
2127
2122 self.FTP_WEI = ftp_wei
2128 self.FTP_WEI = ftp_wei
2123 self.EXP_CODE = exp_code
2129 self.EXP_CODE = exp_code
2124 self.SUB_EXP_CODE = sub_exp_code
2130 self.SUB_EXP_CODE = sub_exp_code
2125 self.PLOT_POS = plot_pos
2131 self.PLOT_POS = plot_pos
2126
2132
2127 self.isConfig = True
2133 self.isConfig = True
2128
2134
2129 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
2135 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
2130
2136
2131 for i in range(self.nplots):
2137 for i in range(self.nplots):
2132 title = self.titleList[i] + ": " +str_datetime
2138 title = self.titleList[i] + ": " +str_datetime
2133 axes = self.axesList[i]
2139 axes = self.axesList[i]
2134 axes.pcolor(x, y, z[i,:].T,
2140 axes.pcolor(x, y, z[i,:].T,
2135 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=self.zminList[i], zmax=self.zmaxList[i],
2141 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=self.zminList[i], zmax=self.zmaxList[i],
2136 xlabel=xlabel, ylabel=ylabel, title=title, colormap=self.cmapList[i],ticksize=9, cblabel='')
2142 xlabel=xlabel, ylabel=ylabel, title=title, colormap=self.cmapList[i],ticksize=9, cblabel='')
2137 self.draw()
2143 self.draw()
2138
2144
2139 if figfile == None:
2145 if figfile == None:
2140 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
2146 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
2141 name = str_datetime
2147 name = str_datetime
2142 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
2148 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
2143 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
2149 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
2144 figfile = self.getFilename(name)
2150 figfile = self.getFilename(name)
2145
2151
2146 self.save(figpath=figpath,
2152 self.save(figpath=figpath,
2147 figfile=figfile,
2153 figfile=figfile,
2148 save=save,
2154 save=save,
2149 ftp=ftp,
2155 ftp=ftp,
2150 wr_period=wr_period,
2156 wr_period=wr_period,
2151 thisDatetime=thisDatetime) No newline at end of file
2157 thisDatetime=thisDatetime)
2158 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1587 +1,1587
1 '''
1 '''
2 Created on Jul 9, 2014
2 Created on Jul 9, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import datetime
7 import datetime
8 import numpy
8 import numpy
9
9
10 from .figure import Figure, isRealtime, isTimeInHourRange
10 from .figure import Figure, isRealtime, isTimeInHourRange
11 from .plotting_codes import *
11 from .plotting_codes import *
12 from schainpy.model.proc.jroproc_base import MPDecorator
12 from schainpy.model.proc.jroproc_base import MPDecorator
13
13
14 from schainpy.utils import log
14 from schainpy.utils import log
15
15
16 @MPDecorator
16 @MPDecorator
17 class SpectraPlot(Figure):
17 class SpectraPlot(Figure):
18
18
19 isConfig = None
19 isConfig = None
20 __nsubplots = None
20 __nsubplots = None
21
21
22 WIDTHPROF = None
22 WIDTHPROF = None
23 HEIGHTPROF = None
23 HEIGHTPROF = None
24 PREFIX = 'spc'
24 PREFIX = 'spc'
25
25
26 def __init__(self):#, **kwargs):
26 def __init__(self):
27 Figure.__init__(self)#, **kwargs)
27 Figure.__init__(self)
28 self.isConfig = False
28 self.isConfig = False
29 self.__nsubplots = 1
29 self.__nsubplots = 1
30 self.WIDTH = 250
30 self.WIDTH = 250
31 self.HEIGHT = 250
31 self.HEIGHT = 250
32 self.WIDTHPROF = 120
32 self.WIDTHPROF = 120
33 self.HEIGHTPROF = 0
33 self.HEIGHTPROF = 0
34 self.counter_imagwr = 0
34 self.counter_imagwr = 0
35
35
36 self.PLOT_CODE = SPEC_CODE
36 self.PLOT_CODE = SPEC_CODE
37
37
38 self.FTP_WEI = None
38 self.FTP_WEI = None
39 self.EXP_CODE = None
39 self.EXP_CODE = None
40 self.SUB_EXP_CODE = None
40 self.SUB_EXP_CODE = None
41 self.PLOT_POS = None
41 self.PLOT_POS = None
42
42
43 self.__xfilter_ena = False
43 self.__xfilter_ena = False
44 self.__yfilter_ena = False
44 self.__yfilter_ena = False
45
45
46 def getSubplots(self):
46 def getSubplots(self):
47
47
48 ncol = int(numpy.sqrt(self.nplots)+0.9)
48 ncol = int(numpy.sqrt(self.nplots)+0.9)
49 nrow = int(self.nplots*1./ncol + 0.9)
49 nrow = int(self.nplots*1./ncol + 0.9)
50
50
51 return nrow, ncol
51 return nrow, ncol
52
52
53 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
53 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
54
54
55 self.__showprofile = showprofile
55 self.__showprofile = showprofile
56 self.nplots = nplots
56 self.nplots = nplots
57
57
58 ncolspan = 1
58 ncolspan = 1
59 colspan = 1
59 colspan = 1
60 if showprofile:
60 if showprofile:
61 ncolspan = 3
61 ncolspan = 3
62 colspan = 2
62 colspan = 2
63 self.__nsubplots = 2
63 self.__nsubplots = 2
64
64
65 self.createFigure(id = id,
65 self.createFigure(id = id,
66 wintitle = wintitle,
66 wintitle = wintitle,
67 widthplot = self.WIDTH + self.WIDTHPROF,
67 widthplot = self.WIDTH + self.WIDTHPROF,
68 heightplot = self.HEIGHT + self.HEIGHTPROF,
68 heightplot = self.HEIGHT + self.HEIGHTPROF,
69 show=show)
69 show=show)
70
70
71 nrow, ncol = self.getSubplots()
71 nrow, ncol = self.getSubplots()
72
72
73 counter = 0
73 counter = 0
74 for y in range(nrow):
74 for y in range(nrow):
75 for x in range(ncol):
75 for x in range(ncol):
76
76
77 if counter >= self.nplots:
77 if counter >= self.nplots:
78 break
78 break
79
79
80 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
80 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
81
81
82 if showprofile:
82 if showprofile:
83 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
83 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
84
84
85 counter += 1
85 counter += 1
86
86
87 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
87 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
88 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
88 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
89 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
89 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
90 server=None, folder=None, username=None, password=None,
90 server=None, folder=None, username=None, password=None,
91 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
91 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
92 xaxis="frequency", colormap='jet', normFactor=None):
92 xaxis="frequency", colormap='jet', normFactor=None):
93
93
94 """
94 """
95
95
96 Input:
96 Input:
97 dataOut :
97 dataOut :
98 id :
98 id :
99 wintitle :
99 wintitle :
100 channelList :
100 channelList :
101 showProfile :
101 showProfile :
102 xmin : None,
102 xmin : None,
103 xmax : None,
103 xmax : None,
104 ymin : None,
104 ymin : None,
105 ymax : None,
105 ymax : None,
106 zmin : None,
106 zmin : None,
107 zmax : None
107 zmax : None
108 """
108 """
109 if dataOut.flagNoData:
109 if dataOut.flagNoData:
110 return dataOut
110 return dataOut
111
111
112 if realtime:
112 if realtime:
113 if not(isRealtime(utcdatatime = dataOut.utctime)):
113 if not(isRealtime(utcdatatime = dataOut.utctime)):
114 print('Skipping this plot function')
114 print('Skipping this plot function')
115 return
115 return
116
116
117 if channelList == None:
117 if channelList == None:
118 channelIndexList = dataOut.channelIndexList
118 channelIndexList = dataOut.channelIndexList
119 else:
119 else:
120 channelIndexList = []
120 channelIndexList = []
121 for channel in channelList:
121 for channel in channelList:
122 if channel not in dataOut.channelList:
122 if channel not in dataOut.channelList:
123 raise ValueError("Channel %d is not in dataOut.channelList" %channel)
123 raise ValueError("Channel %d is not in dataOut.channelList" %channel)
124 channelIndexList.append(dataOut.channelList.index(channel))
124 channelIndexList.append(dataOut.channelList.index(channel))
125
125
126 if normFactor is None:
126 if normFactor is None:
127 factor = dataOut.normFactor
127 factor = dataOut.normFactor
128 else:
128 else:
129 factor = normFactor
129 factor = normFactor
130 if xaxis == "frequency":
130 if xaxis == "frequency":
131 x = dataOut.getFreqRange(1)/1000.
131 x = dataOut.getFreqRange(1)/1000.
132 xlabel = "Frequency (kHz)"
132 xlabel = "Frequency (kHz)"
133
133
134 elif xaxis == "time":
134 elif xaxis == "time":
135 x = dataOut.getAcfRange(1)
135 x = dataOut.getAcfRange(1)
136 xlabel = "Time (ms)"
136 xlabel = "Time (ms)"
137
137
138 else:
138 else:
139 x = dataOut.getVelRange(1)
139 x = dataOut.getVelRange(1)
140 xlabel = "Velocity (m/s)"
140 xlabel = "Velocity (m/s)"
141
141
142 ylabel = "Range (Km)"
142 ylabel = "Range (Km)"
143
143
144 y = dataOut.getHeiRange()
144 y = dataOut.getHeiRange()
145
145
146 z = dataOut.data_spc/factor
146 z = dataOut.data_spc/factor
147 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
147 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
148 zdB = 10*numpy.log10(z)
148 zdB = 10*numpy.log10(z)
149
149
150 avg = numpy.average(z, axis=1)
150 avg = numpy.average(z, axis=1)
151 avgdB = 10*numpy.log10(avg)
151 avgdB = 10*numpy.log10(avg)
152
152
153 noise = dataOut.getNoise()/factor
153 noise = dataOut.getNoise()/factor
154 noisedB = 10*numpy.log10(noise)
154 noisedB = 10*numpy.log10(noise)
155
155
156 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
156 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
157 title = wintitle + " Spectra"
157 title = wintitle + " Spectra"
158 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
158 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
159 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
159 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
160
160
161 if not self.isConfig:
161 if not self.isConfig:
162
162
163 nplots = len(channelIndexList)
163 nplots = len(channelIndexList)
164
164
165 self.setup(id=id,
165 self.setup(id=id,
166 nplots=nplots,
166 nplots=nplots,
167 wintitle=wintitle,
167 wintitle=wintitle,
168 showprofile=showprofile,
168 showprofile=showprofile,
169 show=show)
169 show=show)
170
170
171 if xmin == None: xmin = numpy.nanmin(x)
171 if xmin == None: xmin = numpy.nanmin(x)
172 if xmax == None: xmax = numpy.nanmax(x)
172 if xmax == None: xmax = numpy.nanmax(x)
173 if ymin == None: ymin = numpy.nanmin(y)
173 if ymin == None: ymin = numpy.nanmin(y)
174 if ymax == None: ymax = numpy.nanmax(y)
174 if ymax == None: ymax = numpy.nanmax(y)
175 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
175 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
176 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
176 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
177
177
178 self.FTP_WEI = ftp_wei
178 self.FTP_WEI = ftp_wei
179 self.EXP_CODE = exp_code
179 self.EXP_CODE = exp_code
180 self.SUB_EXP_CODE = sub_exp_code
180 self.SUB_EXP_CODE = sub_exp_code
181 self.PLOT_POS = plot_pos
181 self.PLOT_POS = plot_pos
182
182
183 self.isConfig = True
183 self.isConfig = True
184
184
185 self.setWinTitle(title)
185 self.setWinTitle(title)
186
186
187 for i in range(self.nplots):
187 for i in range(self.nplots):
188 index = channelIndexList[i]
188 index = channelIndexList[i]
189 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
189 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
190 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[index], noisedB[index], str_datetime)
190 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[index], noisedB[index], str_datetime)
191 if len(dataOut.beam.codeList) != 0:
191 if len(dataOut.beam.codeList) != 0:
192 title = "Ch%d:%4.2fdB,%2.2f,%2.2f:%s" %(dataOut.channelList[index], noisedB[index], dataOut.beam.azimuthList[index], dataOut.beam.zenithList[index], str_datetime)
192 title = "Ch%d:%4.2fdB,%2.2f,%2.2f:%s" %(dataOut.channelList[index], noisedB[index], dataOut.beam.azimuthList[index], dataOut.beam.zenithList[index], str_datetime)
193
193
194 axes = self.axesList[i*self.__nsubplots]
194 axes = self.axesList[i*self.__nsubplots]
195 axes.pcolor(x, y, zdB[index,:,:],
195 axes.pcolor(x, y, zdB[index,:,:],
196 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
196 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
197 xlabel=xlabel, ylabel=ylabel, title=title, colormap=colormap,
197 xlabel=xlabel, ylabel=ylabel, title=title, colormap=colormap,
198 ticksize=9, cblabel='')
198 ticksize=9, cblabel='')
199
199
200 if self.__showprofile:
200 if self.__showprofile:
201 axes = self.axesList[i*self.__nsubplots +1]
201 axes = self.axesList[i*self.__nsubplots +1]
202 axes.pline(avgdB[index,:], y,
202 axes.pline(avgdB[index,:], y,
203 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
203 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
204 xlabel='dB', ylabel='', title='',
204 xlabel='dB', ylabel='', title='',
205 ytick_visible=False,
205 ytick_visible=False,
206 grid='x')
206 grid='x')
207
207
208 noiseline = numpy.repeat(noisedB[index], len(y))
208 noiseline = numpy.repeat(noisedB[index], len(y))
209 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
209 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
210
210
211 self.draw()
211 self.draw()
212
212
213 if figfile == None:
213 if figfile == None:
214 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
214 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
215 name = str_datetime
215 name = str_datetime
216 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
216 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
217 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
217 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
218 figfile = self.getFilename(name)
218 figfile = self.getFilename(name)
219
219
220 self.save(figpath=figpath,
220 self.save(figpath=figpath,
221 figfile=figfile,
221 figfile=figfile,
222 save=save,
222 save=save,
223 ftp=ftp,
223 ftp=ftp,
224 wr_period=wr_period,
224 wr_period=wr_period,
225 thisDatetime=thisDatetime)
225 thisDatetime=thisDatetime)
226
226
227 return dataOut
227 return dataOut
228 @MPDecorator
228 @MPDecorator
229 class CrossSpectraPlot(Figure):
229 class CrossSpectraPlot(Figure):
230
230
231 isConfig = None
231 isConfig = None
232 __nsubplots = None
232 __nsubplots = None
233
233
234 WIDTH = None
234 WIDTH = None
235 HEIGHT = None
235 HEIGHT = None
236 WIDTHPROF = None
236 WIDTHPROF = None
237 HEIGHTPROF = None
237 HEIGHTPROF = None
238 PREFIX = 'cspc'
238 PREFIX = 'cspc'
239
239
240 def __init__(self):#, **kwargs):
240 def __init__(self):
241 Figure.__init__(self)#, **kwargs)
241 Figure.__init__(self)
242 self.isConfig = False
242 self.isConfig = False
243 self.__nsubplots = 4
243 self.__nsubplots = 4
244 self.counter_imagwr = 0
244 self.counter_imagwr = 0
245 self.WIDTH = 250
245 self.WIDTH = 250
246 self.HEIGHT = 250
246 self.HEIGHT = 250
247 self.WIDTHPROF = 0
247 self.WIDTHPROF = 0
248 self.HEIGHTPROF = 0
248 self.HEIGHTPROF = 0
249
249
250 self.PLOT_CODE = CROSS_CODE
250 self.PLOT_CODE = CROSS_CODE
251 self.FTP_WEI = None
251 self.FTP_WEI = None
252 self.EXP_CODE = None
252 self.EXP_CODE = None
253 self.SUB_EXP_CODE = None
253 self.SUB_EXP_CODE = None
254 self.PLOT_POS = None
254 self.PLOT_POS = None
255
255
256 def getSubplots(self):
256 def getSubplots(self):
257
257
258 ncol = 4
258 ncol = 4
259 nrow = self.nplots
259 nrow = self.nplots
260
260
261 return nrow, ncol
261 return nrow, ncol
262
262
263 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
263 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
264
264
265 self.__showprofile = showprofile
265 self.__showprofile = showprofile
266 self.nplots = nplots
266 self.nplots = nplots
267
267
268 ncolspan = 1
268 ncolspan = 1
269 colspan = 1
269 colspan = 1
270
270
271 self.createFigure(id = id,
271 self.createFigure(id = id,
272 wintitle = wintitle,
272 wintitle = wintitle,
273 widthplot = self.WIDTH + self.WIDTHPROF,
273 widthplot = self.WIDTH + self.WIDTHPROF,
274 heightplot = self.HEIGHT + self.HEIGHTPROF,
274 heightplot = self.HEIGHT + self.HEIGHTPROF,
275 show=True)
275 show=True)
276
276
277 nrow, ncol = self.getSubplots()
277 nrow, ncol = self.getSubplots()
278
278
279 counter = 0
279 counter = 0
280 for y in range(nrow):
280 for y in range(nrow):
281 for x in range(ncol):
281 for x in range(ncol):
282 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
282 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
283
283
284 counter += 1
284 counter += 1
285
285
286 def run(self, dataOut, id, wintitle="", pairsList=None,
286 def run(self, dataOut, id, wintitle="", pairsList=None,
287 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
287 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
288 coh_min=None, coh_max=None, phase_min=None, phase_max=None,
288 coh_min=None, coh_max=None, phase_min=None, phase_max=None,
289 save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
289 save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
290 power_cmap='jet', coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
290 power_cmap='jet', coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
291 server=None, folder=None, username=None, password=None,
291 server=None, folder=None, username=None, password=None,
292 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None,
292 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None,
293 xaxis='frequency'):
293 xaxis='frequency'):
294
294
295 """
295 """
296
296
297 Input:
297 Input:
298 dataOut :
298 dataOut :
299 id :
299 id :
300 wintitle :
300 wintitle :
301 channelList :
301 channelList :
302 showProfile :
302 showProfile :
303 xmin : None,
303 xmin : None,
304 xmax : None,
304 xmax : None,
305 ymin : None,
305 ymin : None,
306 ymax : None,
306 ymax : None,
307 zmin : None,
307 zmin : None,
308 zmax : None
308 zmax : None
309 """
309 """
310
310
311 if dataOut.flagNoData:
311 if dataOut.flagNoData:
312 return dataOut
312 return dataOut
313
313
314 if pairsList == None:
314 if pairsList == None:
315 pairsIndexList = dataOut.pairsIndexList
315 pairsIndexList = dataOut.pairsIndexList
316 else:
316 else:
317 pairsIndexList = []
317 pairsIndexList = []
318 for pair in pairsList:
318 for pair in pairsList:
319 if pair not in dataOut.pairsList:
319 if pair not in dataOut.pairsList:
320 raise ValueError("Pair %s is not in dataOut.pairsList" %str(pair))
320 raise ValueError("Pair %s is not in dataOut.pairsList" %str(pair))
321 pairsIndexList.append(dataOut.pairsList.index(pair))
321 pairsIndexList.append(dataOut.pairsList.index(pair))
322
322
323 if not pairsIndexList:
323 if not pairsIndexList:
324 return
324 return
325
325
326 if len(pairsIndexList) > 4:
326 if len(pairsIndexList) > 4:
327 pairsIndexList = pairsIndexList[0:4]
327 pairsIndexList = pairsIndexList[0:4]
328
328
329 if normFactor is None:
329 if normFactor is None:
330 factor = dataOut.normFactor
330 factor = dataOut.normFactor
331 else:
331 else:
332 factor = normFactor
332 factor = normFactor
333 x = dataOut.getVelRange(1)
333 x = dataOut.getVelRange(1)
334 y = dataOut.getHeiRange()
334 y = dataOut.getHeiRange()
335 z = dataOut.data_spc[:,:,:]/factor
335 z = dataOut.data_spc[:,:,:]/factor
336 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
336 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
337
337
338 noise = dataOut.noise/factor
338 noise = dataOut.noise/factor
339
339
340 zdB = 10*numpy.log10(z)
340 zdB = 10*numpy.log10(z)
341 noisedB = 10*numpy.log10(noise)
341 noisedB = 10*numpy.log10(noise)
342
342
343 if coh_min == None:
343 if coh_min == None:
344 coh_min = 0.0
344 coh_min = 0.0
345 if coh_max == None:
345 if coh_max == None:
346 coh_max = 1.0
346 coh_max = 1.0
347
347
348 if phase_min == None:
348 if phase_min == None:
349 phase_min = -180
349 phase_min = -180
350 if phase_max == None:
350 if phase_max == None:
351 phase_max = 180
351 phase_max = 180
352
352
353 #thisDatetime = dataOut.datatime
353 #thisDatetime = dataOut.datatime
354 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
354 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
355 title = wintitle + " Cross-Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
355 title = wintitle + " Cross-Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
356 # xlabel = "Velocity (m/s)"
356 # xlabel = "Velocity (m/s)"
357 ylabel = "Range (Km)"
357 ylabel = "Range (Km)"
358
358
359 if xaxis == "frequency":
359 if xaxis == "frequency":
360 x = dataOut.getFreqRange(1)/1000.
360 x = dataOut.getFreqRange(1)/1000.
361 xlabel = "Frequency (kHz)"
361 xlabel = "Frequency (kHz)"
362
362
363 elif xaxis == "time":
363 elif xaxis == "time":
364 x = dataOut.getAcfRange(1)
364 x = dataOut.getAcfRange(1)
365 xlabel = "Time (ms)"
365 xlabel = "Time (ms)"
366
366
367 else:
367 else:
368 x = dataOut.getVelRange(1)
368 x = dataOut.getVelRange(1)
369 xlabel = "Velocity (m/s)"
369 xlabel = "Velocity (m/s)"
370
370
371 if not self.isConfig:
371 if not self.isConfig:
372
372
373 nplots = len(pairsIndexList)
373 nplots = len(pairsIndexList)
374
374
375 self.setup(id=id,
375 self.setup(id=id,
376 nplots=nplots,
376 nplots=nplots,
377 wintitle=wintitle,
377 wintitle=wintitle,
378 showprofile=False,
378 showprofile=False,
379 show=show)
379 show=show)
380
380
381 avg = numpy.abs(numpy.average(z, axis=1))
381 avg = numpy.abs(numpy.average(z, axis=1))
382 avgdB = 10*numpy.log10(avg)
382 avgdB = 10*numpy.log10(avg)
383
383
384 if xmin == None: xmin = numpy.nanmin(x)
384 if xmin == None: xmin = numpy.nanmin(x)
385 if xmax == None: xmax = numpy.nanmax(x)
385 if xmax == None: xmax = numpy.nanmax(x)
386 if ymin == None: ymin = numpy.nanmin(y)
386 if ymin == None: ymin = numpy.nanmin(y)
387 if ymax == None: ymax = numpy.nanmax(y)
387 if ymax == None: ymax = numpy.nanmax(y)
388 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
388 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
389 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
389 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
390
390
391 self.FTP_WEI = ftp_wei
391 self.FTP_WEI = ftp_wei
392 self.EXP_CODE = exp_code
392 self.EXP_CODE = exp_code
393 self.SUB_EXP_CODE = sub_exp_code
393 self.SUB_EXP_CODE = sub_exp_code
394 self.PLOT_POS = plot_pos
394 self.PLOT_POS = plot_pos
395
395
396 self.isConfig = True
396 self.isConfig = True
397
397
398 self.setWinTitle(title)
398 self.setWinTitle(title)
399
399
400 for i in range(self.nplots):
400 for i in range(self.nplots):
401 pair = dataOut.pairsList[pairsIndexList[i]]
401 pair = dataOut.pairsList[pairsIndexList[i]]
402
402
403 chan_index0 = dataOut.channelList.index(pair[0])
403 chan_index0 = dataOut.channelList.index(pair[0])
404 chan_index1 = dataOut.channelList.index(pair[1])
404 chan_index1 = dataOut.channelList.index(pair[1])
405
405
406 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
406 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
407 title = "Ch%d: %4.2fdB: %s" %(pair[0], noisedB[chan_index0], str_datetime)
407 title = "Ch%d: %4.2fdB: %s" %(pair[0], noisedB[chan_index0], str_datetime)
408 zdB = 10.*numpy.log10(dataOut.data_spc[chan_index0,:,:]/factor)
408 zdB = 10.*numpy.log10(dataOut.data_spc[chan_index0,:,:]/factor)
409 axes0 = self.axesList[i*self.__nsubplots]
409 axes0 = self.axesList[i*self.__nsubplots]
410 axes0.pcolor(x, y, zdB,
410 axes0.pcolor(x, y, zdB,
411 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
411 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
412 xlabel=xlabel, ylabel=ylabel, title=title,
412 xlabel=xlabel, ylabel=ylabel, title=title,
413 ticksize=9, colormap=power_cmap, cblabel='')
413 ticksize=9, colormap=power_cmap, cblabel='')
414
414
415 title = "Ch%d: %4.2fdB: %s" %(pair[1], noisedB[chan_index1], str_datetime)
415 title = "Ch%d: %4.2fdB: %s" %(pair[1], noisedB[chan_index1], str_datetime)
416 zdB = 10.*numpy.log10(dataOut.data_spc[chan_index1,:,:]/factor)
416 zdB = 10.*numpy.log10(dataOut.data_spc[chan_index1,:,:]/factor)
417 axes0 = self.axesList[i*self.__nsubplots+1]
417 axes0 = self.axesList[i*self.__nsubplots+1]
418 axes0.pcolor(x, y, zdB,
418 axes0.pcolor(x, y, zdB,
419 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
419 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
420 xlabel=xlabel, ylabel=ylabel, title=title,
420 xlabel=xlabel, ylabel=ylabel, title=title,
421 ticksize=9, colormap=power_cmap, cblabel='')
421 ticksize=9, colormap=power_cmap, cblabel='')
422
422
423 coherenceComplex = dataOut.data_cspc[pairsIndexList[i],:,:]/numpy.sqrt(dataOut.data_spc[chan_index0,:,:]*dataOut.data_spc[chan_index1,:,:])
423 coherenceComplex = dataOut.data_cspc[pairsIndexList[i],:,:]/numpy.sqrt(dataOut.data_spc[chan_index0,:,:]*dataOut.data_spc[chan_index1,:,:])
424 coherence = numpy.abs(coherenceComplex)
424 coherence = numpy.abs(coherenceComplex)
425 # phase = numpy.arctan(-1*coherenceComplex.imag/coherenceComplex.real)*180/numpy.pi
425 # phase = numpy.arctan(-1*coherenceComplex.imag/coherenceComplex.real)*180/numpy.pi
426 phase = numpy.arctan2(coherenceComplex.imag, coherenceComplex.real)*180/numpy.pi
426 phase = numpy.arctan2(coherenceComplex.imag, coherenceComplex.real)*180/numpy.pi
427
427
428 title = "Coherence Ch%d * Ch%d" %(pair[0], pair[1])
428 title = "Coherence Ch%d * Ch%d" %(pair[0], pair[1])
429 axes0 = self.axesList[i*self.__nsubplots+2]
429 axes0 = self.axesList[i*self.__nsubplots+2]
430 axes0.pcolor(x, y, coherence,
430 axes0.pcolor(x, y, coherence,
431 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=coh_min, zmax=coh_max,
431 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=coh_min, zmax=coh_max,
432 xlabel=xlabel, ylabel=ylabel, title=title,
432 xlabel=xlabel, ylabel=ylabel, title=title,
433 ticksize=9, colormap=coherence_cmap, cblabel='')
433 ticksize=9, colormap=coherence_cmap, cblabel='')
434
434
435 title = "Phase Ch%d * Ch%d" %(pair[0], pair[1])
435 title = "Phase Ch%d * Ch%d" %(pair[0], pair[1])
436 axes0 = self.axesList[i*self.__nsubplots+3]
436 axes0 = self.axesList[i*self.__nsubplots+3]
437 axes0.pcolor(x, y, phase,
437 axes0.pcolor(x, y, phase,
438 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=phase_min, zmax=phase_max,
438 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=phase_min, zmax=phase_max,
439 xlabel=xlabel, ylabel=ylabel, title=title,
439 xlabel=xlabel, ylabel=ylabel, title=title,
440 ticksize=9, colormap=phase_cmap, cblabel='')
440 ticksize=9, colormap=phase_cmap, cblabel='')
441
441
442
442
443
443
444 self.draw()
444 self.draw()
445
445
446 self.save(figpath=figpath,
446 self.save(figpath=figpath,
447 figfile=figfile,
447 figfile=figfile,
448 save=save,
448 save=save,
449 ftp=ftp,
449 ftp=ftp,
450 wr_period=wr_period,
450 wr_period=wr_period,
451 thisDatetime=thisDatetime)
451 thisDatetime=thisDatetime)
452
452
453 return dataOut
453 return dataOut
454
454
455 @MPDecorator
455 @MPDecorator
456 class RTIPlot(Figure):
456 class RTIPlot(Figure):
457
457
458 __isConfig = None
458 __isConfig = None
459 __nsubplots = None
459 __nsubplots = None
460
460
461 WIDTHPROF = None
461 WIDTHPROF = None
462 HEIGHTPROF = None
462 HEIGHTPROF = None
463 PREFIX = 'rti'
463 PREFIX = 'rti'
464
464
465 def __init__(self):#, **kwargs):
465 def __init__(self):
466
466
467 Figure.__init__(self)#, **kwargs)
467 Figure.__init__(self)
468 self.timerange = None
468 self.timerange = None
469 self.isConfig = False
469 self.isConfig = False
470 self.__nsubplots = 1
470 self.__nsubplots = 1
471
471
472 self.WIDTH = 800
472 self.WIDTH = 800
473 self.HEIGHT = 180
473 self.HEIGHT = 180
474 self.WIDTHPROF = 120
474 self.WIDTHPROF = 120
475 self.HEIGHTPROF = 0
475 self.HEIGHTPROF = 0
476 self.counter_imagwr = 0
476 self.counter_imagwr = 0
477
477
478 self.PLOT_CODE = RTI_CODE
478 self.PLOT_CODE = RTI_CODE
479
479
480 self.FTP_WEI = None
480 self.FTP_WEI = None
481 self.EXP_CODE = None
481 self.EXP_CODE = None
482 self.SUB_EXP_CODE = None
482 self.SUB_EXP_CODE = None
483 self.PLOT_POS = None
483 self.PLOT_POS = None
484 self.tmin = None
484 self.tmin = None
485 self.tmax = None
485 self.tmax = None
486
486
487 self.xmin = None
487 self.xmin = None
488 self.xmax = None
488 self.xmax = None
489
489
490 self.figfile = None
490 self.figfile = None
491
491
492 def getSubplots(self):
492 def getSubplots(self):
493
493
494 ncol = 1
494 ncol = 1
495 nrow = self.nplots
495 nrow = self.nplots
496
496
497 return nrow, ncol
497 return nrow, ncol
498
498
499 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
499 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
500
500
501 self.__showprofile = showprofile
501 self.__showprofile = showprofile
502 self.nplots = nplots
502 self.nplots = nplots
503
503
504 ncolspan = 1
504 ncolspan = 1
505 colspan = 1
505 colspan = 1
506 if showprofile:
506 if showprofile:
507 ncolspan = 7
507 ncolspan = 7
508 colspan = 6
508 colspan = 6
509 self.__nsubplots = 2
509 self.__nsubplots = 2
510
510
511 self.createFigure(id = id,
511 self.createFigure(id = id,
512 wintitle = wintitle,
512 wintitle = wintitle,
513 widthplot = self.WIDTH + self.WIDTHPROF,
513 widthplot = self.WIDTH + self.WIDTHPROF,
514 heightplot = self.HEIGHT + self.HEIGHTPROF,
514 heightplot = self.HEIGHT + self.HEIGHTPROF,
515 show=show)
515 show=show)
516
516
517 nrow, ncol = self.getSubplots()
517 nrow, ncol = self.getSubplots()
518
518
519 counter = 0
519 counter = 0
520 for y in range(nrow):
520 for y in range(nrow):
521 for x in range(ncol):
521 for x in range(ncol):
522
522
523 if counter >= self.nplots:
523 if counter >= self.nplots:
524 break
524 break
525
525
526 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
526 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
527
527
528 if showprofile:
528 if showprofile:
529 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
529 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
530
530
531 counter += 1
531 counter += 1
532
532
533 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
533 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
534 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
534 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
535 timerange=None, colormap='jet',
535 timerange=None, colormap='jet',
536 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
536 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
537 server=None, folder=None, username=None, password=None,
537 server=None, folder=None, username=None, password=None,
538 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None, HEIGHT=None):
538 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None, HEIGHT=None):
539
539
540 """
540 """
541
541
542 Input:
542 Input:
543 dataOut :
543 dataOut :
544 id :
544 id :
545 wintitle :
545 wintitle :
546 channelList :
546 channelList :
547 showProfile :
547 showProfile :
548 xmin : None,
548 xmin : None,
549 xmax : None,
549 xmax : None,
550 ymin : None,
550 ymin : None,
551 ymax : None,
551 ymax : None,
552 zmin : None,
552 zmin : None,
553 zmax : None
553 zmax : None
554 """
554 """
555 if dataOut.flagNoData:
555 if dataOut.flagNoData:
556 return dataOut
556 return dataOut
557
557
558 #colormap = kwargs.get('colormap', 'jet')
558 #colormap = kwargs.get('colormap', 'jet')
559 if HEIGHT is not None:
559 if HEIGHT is not None:
560 self.HEIGHT = HEIGHT
560 self.HEIGHT = HEIGHT
561
561
562 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
562 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
563 return
563 return
564
564
565 if channelList == None:
565 if channelList == None:
566 channelIndexList = dataOut.channelIndexList
566 channelIndexList = dataOut.channelIndexList
567 else:
567 else:
568 channelIndexList = []
568 channelIndexList = []
569 for channel in channelList:
569 for channel in channelList:
570 if channel not in dataOut.channelList:
570 if channel not in dataOut.channelList:
571 raise ValueError("Channel %d is not in dataOut.channelList")
571 raise ValueError("Channel %d is not in dataOut.channelList")
572 channelIndexList.append(dataOut.channelList.index(channel))
572 channelIndexList.append(dataOut.channelList.index(channel))
573
573
574 if normFactor is None:
574 if normFactor is None:
575 factor = dataOut.normFactor
575 factor = dataOut.normFactor
576 else:
576 else:
577 factor = normFactor
577 factor = normFactor
578
578
579 # factor = dataOut.normFactor
579 #factor = dataOut.normFactor
580 x = dataOut.getTimeRange()
580 x = dataOut.getTimeRange()
581 y = dataOut.getHeiRange()
581 y = dataOut.getHeiRange()
582
582
583 z = dataOut.data_spc/factor
583 z = dataOut.data_spc/factor
584 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
584 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
585 avg = numpy.average(z, axis=1)
585 avg = numpy.average(z, axis=1)
586 avgdB = 10.*numpy.log10(avg)
586 avgdB = 10.*numpy.log10(avg)
587 # avgdB = dataOut.getPower()
587 # avgdB = dataOut.getPower()
588
588
589
589
590 thisDatetime = dataOut.datatime
590 thisDatetime = dataOut.datatime
591 # thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
591 #thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
592 title = wintitle + " RTI" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
592 title = wintitle + " RTI" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
593 xlabel = ""
593 xlabel = ""
594 ylabel = "Range (Km)"
594 ylabel = "Range (Km)"
595
595
596 update_figfile = False
596 update_figfile = False
597
597
598 if self.xmax is not None and dataOut.ltctime >= self.xmax: #yong
598 if self.xmax is not None and dataOut.ltctime >= self.xmax: #yong
599 self.counter_imagwr = wr_period
599 self.counter_imagwr = wr_period
600 self.isConfig = False
600 self.isConfig = False
601 update_figfile = True
601 update_figfile = True
602
602
603 if not self.isConfig:
603 if not self.isConfig:
604
604
605 nplots = len(channelIndexList)
605 nplots = len(channelIndexList)
606
606
607 self.setup(id=id,
607 self.setup(id=id,
608 nplots=nplots,
608 nplots=nplots,
609 wintitle=wintitle,
609 wintitle=wintitle,
610 showprofile=showprofile,
610 showprofile=showprofile,
611 show=show)
611 show=show)
612
612
613 if timerange != None:
613 if timerange != None:
614 self.timerange = timerange
614 self.timerange = timerange
615
615
616 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
616 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
617
617
618 noise = dataOut.noise/factor
618 noise = dataOut.noise/factor
619 noisedB = 10*numpy.log10(noise)
619 noisedB = 10*numpy.log10(noise)
620
620
621 if ymin == None: ymin = numpy.nanmin(y)
621 if ymin == None: ymin = numpy.nanmin(y)
622 if ymax == None: ymax = numpy.nanmax(y)
622 if ymax == None: ymax = numpy.nanmax(y)
623 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
623 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
624 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
624 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
625
625
626 self.FTP_WEI = ftp_wei
626 self.FTP_WEI = ftp_wei
627 self.EXP_CODE = exp_code
627 self.EXP_CODE = exp_code
628 self.SUB_EXP_CODE = sub_exp_code
628 self.SUB_EXP_CODE = sub_exp_code
629 self.PLOT_POS = plot_pos
629 self.PLOT_POS = plot_pos
630
630
631 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
631 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
632 self.isConfig = True
632 self.isConfig = True
633 self.figfile = figfile
633 self.figfile = figfile
634 update_figfile = True
634 update_figfile = True
635
635
636 self.setWinTitle(title)
636 self.setWinTitle(title)
637
637
638 for i in range(self.nplots):
638 for i in range(self.nplots):
639 index = channelIndexList[i]
639 index = channelIndexList[i]
640 title = "Channel %d: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
640 title = "Channel %d: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
641 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
641 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
642 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
642 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
643 axes = self.axesList[i*self.__nsubplots]
643 axes = self.axesList[i*self.__nsubplots]
644 zdB = avgdB[index].reshape((1,-1))
644 zdB = avgdB[index].reshape((1,-1))
645 axes.pcolorbuffer(x, y, zdB,
645 axes.pcolorbuffer(x, y, zdB,
646 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
646 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
647 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
647 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
648 ticksize=9, cblabel='', cbsize="1%", colormap=colormap)
648 ticksize=9, cblabel='', cbsize="1%", colormap=colormap)
649
649
650 if self.__showprofile:
650 if self.__showprofile:
651 axes = self.axesList[i*self.__nsubplots +1]
651 axes = self.axesList[i*self.__nsubplots +1]
652 axes.pline(avgdB[index], y,
652 axes.pline(avgdB[index], y,
653 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
653 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
654 xlabel='dB', ylabel='', title='',
654 xlabel='dB', ylabel='', title='',
655 ytick_visible=False,
655 ytick_visible=False,
656 grid='x')
656 grid='x')
657
657
658 self.draw()
658 self.draw()
659
659
660 self.save(figpath=figpath,
660 self.save(figpath=figpath,
661 figfile=figfile,
661 figfile=figfile,
662 save=save,
662 save=save,
663 ftp=ftp,
663 ftp=ftp,
664 wr_period=wr_period,
664 wr_period=wr_period,
665 thisDatetime=thisDatetime,
665 thisDatetime=thisDatetime,
666 update_figfile=update_figfile)
666 update_figfile=update_figfile)
667 return dataOut
667 return dataOut
668
668
669 @MPDecorator
669 @MPDecorator
670 class CoherenceMap(Figure):
670 class CoherenceMap(Figure):
671 isConfig = None
671 isConfig = None
672 __nsubplots = None
672 __nsubplots = None
673
673
674 WIDTHPROF = None
674 WIDTHPROF = None
675 HEIGHTPROF = None
675 HEIGHTPROF = None
676 PREFIX = 'cmap'
676 PREFIX = 'cmap'
677
677
678 def __init__(self):#, **kwargs):
678 def __init__(self):
679 Figure.__init__(self)#, **kwargs)
679 Figure.__init__(self)
680 self.timerange = 2*60*60
680 self.timerange = 2*60*60
681 self.isConfig = False
681 self.isConfig = False
682 self.__nsubplots = 1
682 self.__nsubplots = 1
683
683
684 self.WIDTH = 800
684 self.WIDTH = 800
685 self.HEIGHT = 180
685 self.HEIGHT = 180
686 self.WIDTHPROF = 120
686 self.WIDTHPROF = 120
687 self.HEIGHTPROF = 0
687 self.HEIGHTPROF = 0
688 self.counter_imagwr = 0
688 self.counter_imagwr = 0
689
689
690 self.PLOT_CODE = COH_CODE
690 self.PLOT_CODE = COH_CODE
691
691
692 self.FTP_WEI = None
692 self.FTP_WEI = None
693 self.EXP_CODE = None
693 self.EXP_CODE = None
694 self.SUB_EXP_CODE = None
694 self.SUB_EXP_CODE = None
695 self.PLOT_POS = None
695 self.PLOT_POS = None
696 self.counter_imagwr = 0
696 self.counter_imagwr = 0
697
697
698 self.xmin = None
698 self.xmin = None
699 self.xmax = None
699 self.xmax = None
700
700
701 def getSubplots(self):
701 def getSubplots(self):
702 ncol = 1
702 ncol = 1
703 nrow = self.nplots*2
703 nrow = self.nplots*2
704
704
705 return nrow, ncol
705 return nrow, ncol
706
706
707 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
707 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
708 self.__showprofile = showprofile
708 self.__showprofile = showprofile
709 self.nplots = nplots
709 self.nplots = nplots
710
710
711 ncolspan = 1
711 ncolspan = 1
712 colspan = 1
712 colspan = 1
713 if showprofile:
713 if showprofile:
714 ncolspan = 7
714 ncolspan = 7
715 colspan = 6
715 colspan = 6
716 self.__nsubplots = 2
716 self.__nsubplots = 2
717
717
718 self.createFigure(id = id,
718 self.createFigure(id = id,
719 wintitle = wintitle,
719 wintitle = wintitle,
720 widthplot = self.WIDTH + self.WIDTHPROF,
720 widthplot = self.WIDTH + self.WIDTHPROF,
721 heightplot = self.HEIGHT + self.HEIGHTPROF,
721 heightplot = self.HEIGHT + self.HEIGHTPROF,
722 show=True)
722 show=True)
723
723
724 nrow, ncol = self.getSubplots()
724 nrow, ncol = self.getSubplots()
725
725
726 for y in range(nrow):
726 for y in range(nrow):
727 for x in range(ncol):
727 for x in range(ncol):
728
728
729 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
729 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
730
730
731 if showprofile:
731 if showprofile:
732 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
732 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
733
733
734 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
734 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
735 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
735 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
736 timerange=None, phase_min=None, phase_max=None,
736 timerange=None, phase_min=None, phase_max=None,
737 save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
737 save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
738 coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
738 coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
739 server=None, folder=None, username=None, password=None,
739 server=None, folder=None, username=None, password=None,
740 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
740 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
741
741
742
742
743 if dataOut.flagNoData:
743 if dataOut.flagNoData:
744 return dataOut
744 return dataOut
745
745
746 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
746 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
747 return
747 return
748
748
749 if pairsList == None:
749 if pairsList == None:
750 pairsIndexList = dataOut.pairsIndexList
750 pairsIndexList = dataOut.pairsIndexList
751 else:
751 else:
752 pairsIndexList = []
752 pairsIndexList = []
753 for pair in pairsList:
753 for pair in pairsList:
754 if pair not in dataOut.pairsList:
754 if pair not in dataOut.pairsList:
755 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
755 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
756 pairsIndexList.append(dataOut.pairsList.index(pair))
756 pairsIndexList.append(dataOut.pairsList.index(pair))
757
757
758 if pairsIndexList == []:
758 if pairsIndexList == []:
759 return
759 return
760
760
761 if len(pairsIndexList) > 4:
761 if len(pairsIndexList) > 4:
762 pairsIndexList = pairsIndexList[0:4]
762 pairsIndexList = pairsIndexList[0:4]
763
763
764 if phase_min == None:
764 if phase_min == None:
765 phase_min = -180
765 phase_min = -180
766 if phase_max == None:
766 if phase_max == None:
767 phase_max = 180
767 phase_max = 180
768
768
769 x = dataOut.getTimeRange()
769 x = dataOut.getTimeRange()
770 y = dataOut.getHeiRange()
770 y = dataOut.getHeiRange()
771
771
772 thisDatetime = dataOut.datatime
772 thisDatetime = dataOut.datatime
773
773
774 title = wintitle + " CoherenceMap" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
774 title = wintitle + " CoherenceMap" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
775 xlabel = ""
775 xlabel = ""
776 ylabel = "Range (Km)"
776 ylabel = "Range (Km)"
777 update_figfile = False
777 update_figfile = False
778
778
779 if not self.isConfig:
779 if not self.isConfig:
780 nplots = len(pairsIndexList)
780 nplots = len(pairsIndexList)
781 self.setup(id=id,
781 self.setup(id=id,
782 nplots=nplots,
782 nplots=nplots,
783 wintitle=wintitle,
783 wintitle=wintitle,
784 showprofile=showprofile,
784 showprofile=showprofile,
785 show=show)
785 show=show)
786
786
787 if timerange != None:
787 if timerange != None:
788 self.timerange = timerange
788 self.timerange = timerange
789
789
790 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
790 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
791
791
792 if ymin == None: ymin = numpy.nanmin(y)
792 if ymin == None: ymin = numpy.nanmin(y)
793 if ymax == None: ymax = numpy.nanmax(y)
793 if ymax == None: ymax = numpy.nanmax(y)
794 if zmin == None: zmin = 0.
794 if zmin == None: zmin = 0.
795 if zmax == None: zmax = 1.
795 if zmax == None: zmax = 1.
796
796
797 self.FTP_WEI = ftp_wei
797 self.FTP_WEI = ftp_wei
798 self.EXP_CODE = exp_code
798 self.EXP_CODE = exp_code
799 self.SUB_EXP_CODE = sub_exp_code
799 self.SUB_EXP_CODE = sub_exp_code
800 self.PLOT_POS = plot_pos
800 self.PLOT_POS = plot_pos
801
801
802 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
802 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
803
803
804 self.isConfig = True
804 self.isConfig = True
805 update_figfile = True
805 update_figfile = True
806
806
807 self.setWinTitle(title)
807 self.setWinTitle(title)
808
808
809 for i in range(self.nplots):
809 for i in range(self.nplots):
810
810
811 pair = dataOut.pairsList[pairsIndexList[i]]
811 pair = dataOut.pairsList[pairsIndexList[i]]
812
812
813 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i],:,:],axis=0)
813 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i],:,:],axis=0)
814 powa = numpy.average(dataOut.data_spc[pair[0],:,:],axis=0)
814 powa = numpy.average(dataOut.data_spc[pair[0],:,:],axis=0)
815 powb = numpy.average(dataOut.data_spc[pair[1],:,:],axis=0)
815 powb = numpy.average(dataOut.data_spc[pair[1],:,:],axis=0)
816
816
817
817
818 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
818 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
819 coherence = numpy.abs(avgcoherenceComplex)
819 coherence = numpy.abs(avgcoherenceComplex)
820
820
821 z = coherence.reshape((1,-1))
821 z = coherence.reshape((1,-1))
822
822
823 counter = 0
823 counter = 0
824
824
825 title = "Coherence Ch%d * Ch%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
825 title = "Coherence Ch%d * Ch%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
826 axes = self.axesList[i*self.__nsubplots*2]
826 axes = self.axesList[i*self.__nsubplots*2]
827 axes.pcolorbuffer(x, y, z,
827 axes.pcolorbuffer(x, y, z,
828 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
828 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
829 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
829 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
830 ticksize=9, cblabel='', colormap=coherence_cmap, cbsize="1%")
830 ticksize=9, cblabel='', colormap=coherence_cmap, cbsize="1%")
831
831
832 if self.__showprofile:
832 if self.__showprofile:
833 counter += 1
833 counter += 1
834 axes = self.axesList[i*self.__nsubplots*2 + counter]
834 axes = self.axesList[i*self.__nsubplots*2 + counter]
835 axes.pline(coherence, y,
835 axes.pline(coherence, y,
836 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
836 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
837 xlabel='', ylabel='', title='', ticksize=7,
837 xlabel='', ylabel='', title='', ticksize=7,
838 ytick_visible=False, nxticks=5,
838 ytick_visible=False, nxticks=5,
839 grid='x')
839 grid='x')
840
840
841 counter += 1
841 counter += 1
842
842
843 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
843 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
844
844
845 z = phase.reshape((1,-1))
845 z = phase.reshape((1,-1))
846
846
847 title = "Phase Ch%d * Ch%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
847 title = "Phase Ch%d * Ch%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
848 axes = self.axesList[i*self.__nsubplots*2 + counter]
848 axes = self.axesList[i*self.__nsubplots*2 + counter]
849 axes.pcolorbuffer(x, y, z,
849 axes.pcolorbuffer(x, y, z,
850 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=phase_min, zmax=phase_max,
850 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=phase_min, zmax=phase_max,
851 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
851 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
852 ticksize=9, cblabel='', colormap=phase_cmap, cbsize="1%")
852 ticksize=9, cblabel='', colormap=phase_cmap, cbsize="1%")
853
853
854 if self.__showprofile:
854 if self.__showprofile:
855 counter += 1
855 counter += 1
856 axes = self.axesList[i*self.__nsubplots*2 + counter]
856 axes = self.axesList[i*self.__nsubplots*2 + counter]
857 axes.pline(phase, y,
857 axes.pline(phase, y,
858 xmin=phase_min, xmax=phase_max, ymin=ymin, ymax=ymax,
858 xmin=phase_min, xmax=phase_max, ymin=ymin, ymax=ymax,
859 xlabel='', ylabel='', title='', ticksize=7,
859 xlabel='', ylabel='', title='', ticksize=7,
860 ytick_visible=False, nxticks=4,
860 ytick_visible=False, nxticks=4,
861 grid='x')
861 grid='x')
862
862
863 self.draw()
863 self.draw()
864
864
865 if dataOut.ltctime >= self.xmax:
865 if dataOut.ltctime >= self.xmax:
866 self.counter_imagwr = wr_period
866 self.counter_imagwr = wr_period
867 self.isConfig = False
867 self.isConfig = False
868 update_figfile = True
868 update_figfile = True
869
869
870 self.save(figpath=figpath,
870 self.save(figpath=figpath,
871 figfile=figfile,
871 figfile=figfile,
872 save=save,
872 save=save,
873 ftp=ftp,
873 ftp=ftp,
874 wr_period=wr_period,
874 wr_period=wr_period,
875 thisDatetime=thisDatetime,
875 thisDatetime=thisDatetime,
876 update_figfile=update_figfile)
876 update_figfile=update_figfile)
877
877
878 return dataOut
878 return dataOut
879
879
880 @MPDecorator
880 @MPDecorator
881 class PowerProfilePlot(Figure):
881 class PowerProfilePlot(Figure):
882
882
883 isConfig = None
883 isConfig = None
884 __nsubplots = None
884 __nsubplots = None
885
885
886 WIDTHPROF = None
886 WIDTHPROF = None
887 HEIGHTPROF = None
887 HEIGHTPROF = None
888 PREFIX = 'spcprofile'
888 PREFIX = 'spcprofile'
889
889
890 def __init__(self):#, **kwargs):
890 def __init__(self):
891 Figure.__init__(self)#, **kwargs)
891 Figure.__init__(self)
892 self.isConfig = False
892 self.isConfig = False
893 self.__nsubplots = 1
893 self.__nsubplots = 1
894
894
895 self.PLOT_CODE = POWER_CODE
895 self.PLOT_CODE = POWER_CODE
896
896
897 self.WIDTH = 300
897 self.WIDTH = 300
898 self.HEIGHT = 500
898 self.HEIGHT = 500
899 self.counter_imagwr = 0
899 self.counter_imagwr = 0
900
900
901 def getSubplots(self):
901 def getSubplots(self):
902 ncol = 1
902 ncol = 1
903 nrow = 1
903 nrow = 1
904
904
905 return nrow, ncol
905 return nrow, ncol
906
906
907 def setup(self, id, nplots, wintitle, show):
907 def setup(self, id, nplots, wintitle, show):
908
908
909 self.nplots = nplots
909 self.nplots = nplots
910
910
911 ncolspan = 1
911 ncolspan = 1
912 colspan = 1
912 colspan = 1
913
913
914 self.createFigure(id = id,
914 self.createFigure(id = id,
915 wintitle = wintitle,
915 wintitle = wintitle,
916 widthplot = self.WIDTH,
916 widthplot = self.WIDTH,
917 heightplot = self.HEIGHT,
917 heightplot = self.HEIGHT,
918 show=show)
918 show=show)
919
919
920 nrow, ncol = self.getSubplots()
920 nrow, ncol = self.getSubplots()
921
921
922 counter = 0
922 counter = 0
923 for y in range(nrow):
923 for y in range(nrow):
924 for x in range(ncol):
924 for x in range(ncol):
925 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
925 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
926
926
927 def run(self, dataOut, id, wintitle="", channelList=None,
927 def run(self, dataOut, id, wintitle="", channelList=None,
928 xmin=None, xmax=None, ymin=None, ymax=None,
928 xmin=None, xmax=None, ymin=None, ymax=None,
929 save=False, figpath='./', figfile=None, show=True,
929 save=False, figpath='./', figfile=None, show=True,
930 ftp=False, wr_period=1, server=None,
930 ftp=False, wr_period=1, server=None,
931 folder=None, username=None, password=None):
931 folder=None, username=None, password=None):
932
932
933 if dataOut.flagNoData:
933 if dataOut.flagNoData:
934 return dataOut
934 return dataOut
935
935
936
936
937 if channelList == None:
937 if channelList == None:
938 channelIndexList = dataOut.channelIndexList
938 channelIndexList = dataOut.channelIndexList
939 channelList = dataOut.channelList
939 channelList = dataOut.channelList
940 else:
940 else:
941 channelIndexList = []
941 channelIndexList = []
942 for channel in channelList:
942 for channel in channelList:
943 if channel not in dataOut.channelList:
943 if channel not in dataOut.channelList:
944 raise ValueError("Channel %d is not in dataOut.channelList")
944 raise ValueError("Channel %d is not in dataOut.channelList")
945 channelIndexList.append(dataOut.channelList.index(channel))
945 channelIndexList.append(dataOut.channelList.index(channel))
946
946
947 factor = dataOut.normFactor
947 factor = dataOut.normFactor
948
948
949 y = dataOut.getHeiRange()
949 y = dataOut.getHeiRange()
950
950
951 #for voltage
951 #for voltage
952 if dataOut.type == 'Voltage':
952 if dataOut.type == 'Voltage':
953 x = dataOut.data[channelIndexList,:] * numpy.conjugate(dataOut.data[channelIndexList,:])
953 x = dataOut.data[channelIndexList,:] * numpy.conjugate(dataOut.data[channelIndexList,:])
954 x = x.real
954 x = x.real
955 x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
955 x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
956
956
957 #for spectra
957 #for spectra
958 if dataOut.type == 'Spectra':
958 if dataOut.type == 'Spectra':
959 x = dataOut.data_spc[channelIndexList,:,:]/factor
959 x = dataOut.data_spc[channelIndexList,:,:]/factor
960 x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
960 x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
961 x = numpy.average(x, axis=1)
961 x = numpy.average(x, axis=1)
962
962
963
963
964 xdB = 10*numpy.log10(x)
964 xdB = 10*numpy.log10(x)
965
965
966 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
966 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
967 title = wintitle + " Power Profile %s" %(thisDatetime.strftime("%d-%b-%Y"))
967 title = wintitle + " Power Profile %s" %(thisDatetime.strftime("%d-%b-%Y"))
968 xlabel = "dB"
968 xlabel = "dB"
969 ylabel = "Range (Km)"
969 ylabel = "Range (Km)"
970
970
971 if not self.isConfig:
971 if not self.isConfig:
972
972
973 nplots = 1
973 nplots = 1
974
974
975 self.setup(id=id,
975 self.setup(id=id,
976 nplots=nplots,
976 nplots=nplots,
977 wintitle=wintitle,
977 wintitle=wintitle,
978 show=show)
978 show=show)
979
979
980 if ymin == None: ymin = numpy.nanmin(y)
980 if ymin == None: ymin = numpy.nanmin(y)
981 if ymax == None: ymax = numpy.nanmax(y)
981 if ymax == None: ymax = numpy.nanmax(y)
982 if xmin == None: xmin = numpy.nanmin(xdB)*0.9
982 if xmin == None: xmin = numpy.nanmin(xdB)*0.9
983 if xmax == None: xmax = numpy.nanmax(xdB)*1.1
983 if xmax == None: xmax = numpy.nanmax(xdB)*1.1
984
984
985 self.isConfig = True
985 self.isConfig = True
986
986
987 self.setWinTitle(title)
987 self.setWinTitle(title)
988
988
989 title = "Power Profile: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
989 title = "Power Profile: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
990 axes = self.axesList[0]
990 axes = self.axesList[0]
991
991
992 legendlabels = ["channel %d"%x for x in channelList]
992 legendlabels = ["channel %d"%x for x in channelList]
993 axes.pmultiline(xdB, y,
993 axes.pmultiline(xdB, y,
994 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
994 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
995 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
995 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
996 ytick_visible=True, nxticks=5,
996 ytick_visible=True, nxticks=5,
997 grid='x')
997 grid='x')
998
998
999 self.draw()
999 self.draw()
1000
1000
1001 self.save(figpath=figpath,
1001 self.save(figpath=figpath,
1002 figfile=figfile,
1002 figfile=figfile,
1003 save=save,
1003 save=save,
1004 ftp=ftp,
1004 ftp=ftp,
1005 wr_period=wr_period,
1005 wr_period=wr_period,
1006 thisDatetime=thisDatetime)
1006 thisDatetime=thisDatetime)
1007
1007
1008 return dataOut
1008 return dataOut
1009
1009
1010 @MPDecorator
1010 @MPDecorator
1011 class SpectraCutPlot(Figure):
1011 class SpectraCutPlot(Figure):
1012
1012
1013 isConfig = None
1013 isConfig = None
1014 __nsubplots = None
1014 __nsubplots = None
1015
1015
1016 WIDTHPROF = None
1016 WIDTHPROF = None
1017 HEIGHTPROF = None
1017 HEIGHTPROF = None
1018 PREFIX = 'spc_cut'
1018 PREFIX = 'spc_cut'
1019
1019
1020 def __init__(self):#, **kwargs):
1020 def __init__(self):
1021 Figure.__init__(self)#, **kwargs)
1021 Figure.__init__(self)
1022 self.isConfig = False
1022 self.isConfig = False
1023 self.__nsubplots = 1
1023 self.__nsubplots = 1
1024
1024
1025 self.PLOT_CODE = POWER_CODE
1025 self.PLOT_CODE = POWER_CODE
1026
1026
1027 self.WIDTH = 700
1027 self.WIDTH = 700
1028 self.HEIGHT = 500
1028 self.HEIGHT = 500
1029 self.counter_imagwr = 0
1029 self.counter_imagwr = 0
1030
1030
1031 def getSubplots(self):
1031 def getSubplots(self):
1032 ncol = 1
1032 ncol = 1
1033 nrow = 1
1033 nrow = 1
1034
1034
1035 return nrow, ncol
1035 return nrow, ncol
1036
1036
1037 def setup(self, id, nplots, wintitle, show):
1037 def setup(self, id, nplots, wintitle, show):
1038
1038
1039 self.nplots = nplots
1039 self.nplots = nplots
1040
1040
1041 ncolspan = 1
1041 ncolspan = 1
1042 colspan = 1
1042 colspan = 1
1043
1043
1044 self.createFigure(id = id,
1044 self.createFigure(id = id,
1045 wintitle = wintitle,
1045 wintitle = wintitle,
1046 widthplot = self.WIDTH,
1046 widthplot = self.WIDTH,
1047 heightplot = self.HEIGHT,
1047 heightplot = self.HEIGHT,
1048 show=show)
1048 show=show)
1049
1049
1050 nrow, ncol = self.getSubplots()
1050 nrow, ncol = self.getSubplots()
1051
1051
1052 counter = 0
1052 counter = 0
1053 for y in range(nrow):
1053 for y in range(nrow):
1054 for x in range(ncol):
1054 for x in range(ncol):
1055 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1055 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1056
1056
1057 def run(self, dataOut, id, wintitle="", channelList=None,
1057 def run(self, dataOut, id, wintitle="", channelList=None,
1058 xmin=None, xmax=None, ymin=None, ymax=None,
1058 xmin=None, xmax=None, ymin=None, ymax=None,
1059 save=False, figpath='./', figfile=None, show=True,
1059 save=False, figpath='./', figfile=None, show=True,
1060 ftp=False, wr_period=1, server=None,
1060 ftp=False, wr_period=1, server=None,
1061 folder=None, username=None, password=None,
1061 folder=None, username=None, password=None,
1062 xaxis="frequency"):
1062 xaxis="frequency"):
1063
1063
1064 if dataOut.flagNoData:
1064 if dataOut.flagNoData:
1065 return dataOut
1065 return dataOut
1066
1066
1067 if channelList == None:
1067 if channelList == None:
1068 channelIndexList = dataOut.channelIndexList
1068 channelIndexList = dataOut.channelIndexList
1069 channelList = dataOut.channelList
1069 channelList = dataOut.channelList
1070 else:
1070 else:
1071 channelIndexList = []
1071 channelIndexList = []
1072 for channel in channelList:
1072 for channel in channelList:
1073 if channel not in dataOut.channelList:
1073 if channel not in dataOut.channelList:
1074 raise ValueError("Channel %d is not in dataOut.channelList")
1074 raise ValueError("Channel %d is not in dataOut.channelList")
1075 channelIndexList.append(dataOut.channelList.index(channel))
1075 channelIndexList.append(dataOut.channelList.index(channel))
1076
1076
1077 factor = dataOut.normFactor
1077 factor = dataOut.normFactor
1078
1078
1079 y = dataOut.getHeiRange()
1079 y = dataOut.getHeiRange()
1080
1080
1081 z = dataOut.data_spc/factor
1081 z = dataOut.data_spc/factor
1082 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
1082 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
1083
1083
1084 hei_index = numpy.arange(25)*3 + 20
1084 hei_index = numpy.arange(25)*3 + 20
1085
1085
1086 if xaxis == "frequency":
1086 if xaxis == "frequency":
1087 x = dataOut.getFreqRange()/1000.
1087 x = dataOut.getFreqRange()/1000.
1088 zdB = 10*numpy.log10(z[0,:,hei_index])
1088 zdB = 10*numpy.log10(z[0,:,hei_index])
1089 xlabel = "Frequency (kHz)"
1089 xlabel = "Frequency (kHz)"
1090 ylabel = "Power (dB)"
1090 ylabel = "Power (dB)"
1091
1091
1092 elif xaxis == "time":
1092 elif xaxis == "time":
1093 x = dataOut.getAcfRange()
1093 x = dataOut.getAcfRange()
1094 zdB = z[0,:,hei_index]
1094 zdB = z[0,:,hei_index]
1095 xlabel = "Time (ms)"
1095 xlabel = "Time (ms)"
1096 ylabel = "ACF"
1096 ylabel = "ACF"
1097
1097
1098 else:
1098 else:
1099 x = dataOut.getVelRange()
1099 x = dataOut.getVelRange()
1100 zdB = 10*numpy.log10(z[0,:,hei_index])
1100 zdB = 10*numpy.log10(z[0,:,hei_index])
1101 xlabel = "Velocity (m/s)"
1101 xlabel = "Velocity (m/s)"
1102 ylabel = "Power (dB)"
1102 ylabel = "Power (dB)"
1103
1103
1104 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1104 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1105 title = wintitle + " Range Cuts %s" %(thisDatetime.strftime("%d-%b-%Y"))
1105 title = wintitle + " Range Cuts %s" %(thisDatetime.strftime("%d-%b-%Y"))
1106
1106
1107 if not self.isConfig:
1107 if not self.isConfig:
1108
1108
1109 nplots = 1
1109 nplots = 1
1110
1110
1111 self.setup(id=id,
1111 self.setup(id=id,
1112 nplots=nplots,
1112 nplots=nplots,
1113 wintitle=wintitle,
1113 wintitle=wintitle,
1114 show=show)
1114 show=show)
1115
1115
1116 if xmin == None: xmin = numpy.nanmin(x)*0.9
1116 if xmin == None: xmin = numpy.nanmin(x)*0.9
1117 if xmax == None: xmax = numpy.nanmax(x)*1.1
1117 if xmax == None: xmax = numpy.nanmax(x)*1.1
1118 if ymin == None: ymin = numpy.nanmin(zdB)
1118 if ymin == None: ymin = numpy.nanmin(zdB)
1119 if ymax == None: ymax = numpy.nanmax(zdB)
1119 if ymax == None: ymax = numpy.nanmax(zdB)
1120
1120
1121 self.isConfig = True
1121 self.isConfig = True
1122
1122
1123 self.setWinTitle(title)
1123 self.setWinTitle(title)
1124
1124
1125 title = "Spectra Cuts: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
1125 title = "Spectra Cuts: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
1126 axes = self.axesList[0]
1126 axes = self.axesList[0]
1127
1127
1128 legendlabels = ["Range = %dKm" %y[i] for i in hei_index]
1128 legendlabels = ["Range = %dKm" %y[i] for i in hei_index]
1129
1129
1130 axes.pmultilineyaxis( x, zdB,
1130 axes.pmultilineyaxis( x, zdB,
1131 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1131 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1132 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
1132 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
1133 ytick_visible=True, nxticks=5,
1133 ytick_visible=True, nxticks=5,
1134 grid='x')
1134 grid='x')
1135
1135
1136 self.draw()
1136 self.draw()
1137
1137
1138 self.save(figpath=figpath,
1138 self.save(figpath=figpath,
1139 figfile=figfile,
1139 figfile=figfile,
1140 save=save,
1140 save=save,
1141 ftp=ftp,
1141 ftp=ftp,
1142 wr_period=wr_period,
1142 wr_period=wr_period,
1143 thisDatetime=thisDatetime)
1143 thisDatetime=thisDatetime)
1144
1144
1145 return dataOut
1145 return dataOut
1146
1146
1147 @MPDecorator
1147 @MPDecorator
1148 class Noise(Figure):
1148 class Noise(Figure):
1149
1149
1150 isConfig = None
1150 isConfig = None
1151 __nsubplots = None
1151 __nsubplots = None
1152
1152
1153 PREFIX = 'noise'
1153 PREFIX = 'noise'
1154
1154
1155
1155
1156 def __init__(self):#, **kwargs):
1156 def __init__(self):
1157 Figure.__init__(self)#, **kwargs)
1157 Figure.__init__(self)
1158 self.timerange = 24*60*60
1158 self.timerange = 24*60*60
1159 self.isConfig = False
1159 self.isConfig = False
1160 self.__nsubplots = 1
1160 self.__nsubplots = 1
1161 self.counter_imagwr = 0
1161 self.counter_imagwr = 0
1162 self.WIDTH = 800
1162 self.WIDTH = 800
1163 self.HEIGHT = 400
1163 self.HEIGHT = 400
1164 self.WIDTHPROF = 120
1164 self.WIDTHPROF = 120
1165 self.HEIGHTPROF = 0
1165 self.HEIGHTPROF = 0
1166 self.xdata = None
1166 self.xdata = None
1167 self.ydata = None
1167 self.ydata = None
1168
1168
1169 self.PLOT_CODE = NOISE_CODE
1169 self.PLOT_CODE = NOISE_CODE
1170
1170
1171 self.FTP_WEI = None
1171 self.FTP_WEI = None
1172 self.EXP_CODE = None
1172 self.EXP_CODE = None
1173 self.SUB_EXP_CODE = None
1173 self.SUB_EXP_CODE = None
1174 self.PLOT_POS = None
1174 self.PLOT_POS = None
1175 self.figfile = None
1175 self.figfile = None
1176
1176
1177 self.xmin = None
1177 self.xmin = None
1178 self.xmax = None
1178 self.xmax = None
1179
1179
1180 def getSubplots(self):
1180 def getSubplots(self):
1181
1181
1182 ncol = 1
1182 ncol = 1
1183 nrow = 1
1183 nrow = 1
1184
1184
1185 return nrow, ncol
1185 return nrow, ncol
1186
1186
1187 def openfile(self, filename):
1187 def openfile(self, filename):
1188 dirname = os.path.dirname(filename)
1188 dirname = os.path.dirname(filename)
1189
1189
1190 if not os.path.exists(dirname):
1190 if not os.path.exists(dirname):
1191 os.mkdir(dirname)
1191 os.mkdir(dirname)
1192
1192
1193 f = open(filename,'w+')
1193 f = open(filename,'w+')
1194 f.write('\n\n')
1194 f.write('\n\n')
1195 f.write('JICAMARCA RADIO OBSERVATORY - Noise \n')
1195 f.write('JICAMARCA RADIO OBSERVATORY - Noise \n')
1196 f.write('DD MM YYYY HH MM SS Channel0 Channel1 Channel2 Channel3\n\n' )
1196 f.write('DD MM YYYY HH MM SS Channel0 Channel1 Channel2 Channel3\n\n' )
1197 f.close()
1197 f.close()
1198
1198
1199 def save_data(self, filename_phase, data, data_datetime):
1199 def save_data(self, filename_phase, data, data_datetime):
1200
1200
1201 f=open(filename_phase,'a')
1201 f=open(filename_phase,'a')
1202
1202
1203 timetuple_data = data_datetime.timetuple()
1203 timetuple_data = data_datetime.timetuple()
1204 day = str(timetuple_data.tm_mday)
1204 day = str(timetuple_data.tm_mday)
1205 month = str(timetuple_data.tm_mon)
1205 month = str(timetuple_data.tm_mon)
1206 year = str(timetuple_data.tm_year)
1206 year = str(timetuple_data.tm_year)
1207 hour = str(timetuple_data.tm_hour)
1207 hour = str(timetuple_data.tm_hour)
1208 minute = str(timetuple_data.tm_min)
1208 minute = str(timetuple_data.tm_min)
1209 second = str(timetuple_data.tm_sec)
1209 second = str(timetuple_data.tm_sec)
1210
1210
1211 data_msg = ''
1211 data_msg = ''
1212 for i in range(len(data)):
1212 for i in range(len(data)):
1213 data_msg += str(data[i]) + ' '
1213 data_msg += str(data[i]) + ' '
1214
1214
1215 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' ' + data_msg + '\n')
1215 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' ' + data_msg + '\n')
1216 f.close()
1216 f.close()
1217
1217
1218
1218
1219 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1219 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1220
1220
1221 self.__showprofile = showprofile
1221 self.__showprofile = showprofile
1222 self.nplots = nplots
1222 self.nplots = nplots
1223
1223
1224 ncolspan = 7
1224 ncolspan = 7
1225 colspan = 6
1225 colspan = 6
1226 self.__nsubplots = 2
1226 self.__nsubplots = 2
1227
1227
1228 self.createFigure(id = id,
1228 self.createFigure(id = id,
1229 wintitle = wintitle,
1229 wintitle = wintitle,
1230 widthplot = self.WIDTH+self.WIDTHPROF,
1230 widthplot = self.WIDTH+self.WIDTHPROF,
1231 heightplot = self.HEIGHT+self.HEIGHTPROF,
1231 heightplot = self.HEIGHT+self.HEIGHTPROF,
1232 show=show)
1232 show=show)
1233
1233
1234 nrow, ncol = self.getSubplots()
1234 nrow, ncol = self.getSubplots()
1235
1235
1236 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1236 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1237
1237
1238
1238
1239 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
1239 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
1240 xmin=None, xmax=None, ymin=None, ymax=None,
1240 xmin=None, xmax=None, ymin=None, ymax=None,
1241 timerange=None,
1241 timerange=None,
1242 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1242 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1243 server=None, folder=None, username=None, password=None,
1243 server=None, folder=None, username=None, password=None,
1244 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1244 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1245
1245
1246 if dataOut.flagNoData:
1246 if dataOut.flagNoData:
1247 return dataOut
1247 return dataOut
1248
1248
1249 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
1249 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
1250 return
1250 return
1251
1251
1252 if channelList == None:
1252 if channelList == None:
1253 channelIndexList = dataOut.channelIndexList
1253 channelIndexList = dataOut.channelIndexList
1254 channelList = dataOut.channelList
1254 channelList = dataOut.channelList
1255 else:
1255 else:
1256 channelIndexList = []
1256 channelIndexList = []
1257 for channel in channelList:
1257 for channel in channelList:
1258 if channel not in dataOut.channelList:
1258 if channel not in dataOut.channelList:
1259 raise ValueError("Channel %d is not in dataOut.channelList")
1259 raise ValueError("Channel %d is not in dataOut.channelList")
1260 channelIndexList.append(dataOut.channelList.index(channel))
1260 channelIndexList.append(dataOut.channelList.index(channel))
1261
1261
1262 x = dataOut.getTimeRange()
1262 x = dataOut.getTimeRange()
1263 #y = dataOut.getHeiRange()
1263 #y = dataOut.getHeiRange()
1264 factor = dataOut.normFactor
1264 factor = dataOut.normFactor
1265 noise = dataOut.noise[channelIndexList]/factor
1265 noise = dataOut.noise[channelIndexList]/factor
1266 noisedB = 10*numpy.log10(noise)
1266 noisedB = 10*numpy.log10(noise)
1267
1267
1268 thisDatetime = dataOut.datatime
1268 thisDatetime = dataOut.datatime
1269
1269
1270 title = wintitle + " Noise" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1270 title = wintitle + " Noise" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1271 xlabel = ""
1271 xlabel = ""
1272 ylabel = "Intensity (dB)"
1272 ylabel = "Intensity (dB)"
1273 update_figfile = False
1273 update_figfile = False
1274
1274
1275 if not self.isConfig:
1275 if not self.isConfig:
1276
1276
1277 nplots = 1
1277 nplots = 1
1278
1278
1279 self.setup(id=id,
1279 self.setup(id=id,
1280 nplots=nplots,
1280 nplots=nplots,
1281 wintitle=wintitle,
1281 wintitle=wintitle,
1282 showprofile=showprofile,
1282 showprofile=showprofile,
1283 show=show)
1283 show=show)
1284
1284
1285 if timerange != None:
1285 if timerange != None:
1286 self.timerange = timerange
1286 self.timerange = timerange
1287
1287
1288 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1288 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1289
1289
1290 if ymin == None: ymin = numpy.floor(numpy.nanmin(noisedB)) - 10.0
1290 if ymin == None: ymin = numpy.floor(numpy.nanmin(noisedB)) - 10.0
1291 if ymax == None: ymax = numpy.nanmax(noisedB) + 10.0
1291 if ymax == None: ymax = numpy.nanmax(noisedB) + 10.0
1292
1292
1293 self.FTP_WEI = ftp_wei
1293 self.FTP_WEI = ftp_wei
1294 self.EXP_CODE = exp_code
1294 self.EXP_CODE = exp_code
1295 self.SUB_EXP_CODE = sub_exp_code
1295 self.SUB_EXP_CODE = sub_exp_code
1296 self.PLOT_POS = plot_pos
1296 self.PLOT_POS = plot_pos
1297
1297
1298
1298
1299 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1299 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1300 self.isConfig = True
1300 self.isConfig = True
1301 self.figfile = figfile
1301 self.figfile = figfile
1302 self.xdata = numpy.array([])
1302 self.xdata = numpy.array([])
1303 self.ydata = numpy.array([])
1303 self.ydata = numpy.array([])
1304
1304
1305 update_figfile = True
1305 update_figfile = True
1306
1306
1307 #open file beacon phase
1307 #open file beacon phase
1308 path = '%s%03d' %(self.PREFIX, self.id)
1308 path = '%s%03d' %(self.PREFIX, self.id)
1309 noise_file = os.path.join(path,'%s.txt'%self.name)
1309 noise_file = os.path.join(path,'%s.txt'%self.name)
1310 self.filename_noise = os.path.join(figpath,noise_file)
1310 self.filename_noise = os.path.join(figpath,noise_file)
1311
1311
1312 self.setWinTitle(title)
1312 self.setWinTitle(title)
1313
1313
1314 title = "Noise %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1314 title = "Noise %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1315
1315
1316 legendlabels = ["channel %d"%(idchannel) for idchannel in channelList]
1316 legendlabels = ["channel %d"%(idchannel) for idchannel in channelList]
1317 axes = self.axesList[0]
1317 axes = self.axesList[0]
1318
1318
1319 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1319 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1320
1320
1321 if len(self.ydata)==0:
1321 if len(self.ydata)==0:
1322 self.ydata = noisedB.reshape(-1,1)
1322 self.ydata = noisedB.reshape(-1,1)
1323 else:
1323 else:
1324 self.ydata = numpy.hstack((self.ydata, noisedB.reshape(-1,1)))
1324 self.ydata = numpy.hstack((self.ydata, noisedB.reshape(-1,1)))
1325
1325
1326
1326
1327 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1327 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1328 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1328 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1329 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1329 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1330 XAxisAsTime=True, grid='both'
1330 XAxisAsTime=True, grid='both'
1331 )
1331 )
1332
1332
1333 self.draw()
1333 self.draw()
1334
1334
1335 if dataOut.ltctime >= self.xmax:
1335 if dataOut.ltctime >= self.xmax:
1336 self.counter_imagwr = wr_period
1336 self.counter_imagwr = wr_period
1337 self.isConfig = False
1337 self.isConfig = False
1338 update_figfile = True
1338 update_figfile = True
1339
1339
1340 self.save(figpath=figpath,
1340 self.save(figpath=figpath,
1341 figfile=figfile,
1341 figfile=figfile,
1342 save=save,
1342 save=save,
1343 ftp=ftp,
1343 ftp=ftp,
1344 wr_period=wr_period,
1344 wr_period=wr_period,
1345 thisDatetime=thisDatetime,
1345 thisDatetime=thisDatetime,
1346 update_figfile=update_figfile)
1346 update_figfile=update_figfile)
1347
1347
1348 #store data beacon phase
1348 #store data beacon phase
1349 if save:
1349 if save:
1350 self.save_data(self.filename_noise, noisedB, thisDatetime)
1350 self.save_data(self.filename_noise, noisedB, thisDatetime)
1351
1351
1352 return dataOut
1352 return dataOut
1353
1353
1354 @MPDecorator
1354 @MPDecorator
1355 class BeaconPhase(Figure):
1355 class BeaconPhase(Figure):
1356
1356
1357 __isConfig = None
1357 __isConfig = None
1358 __nsubplots = None
1358 __nsubplots = None
1359
1359
1360 PREFIX = 'beacon_phase'
1360 PREFIX = 'beacon_phase'
1361
1361
1362 def __init__(self):#, **kwargs):
1362 def __init__(self):
1363 Figure.__init__(self)#, **kwargs)
1363 Figure.__init__(self)
1364 self.timerange = 24*60*60
1364 self.timerange = 24*60*60
1365 self.isConfig = False
1365 self.isConfig = False
1366 self.__nsubplots = 1
1366 self.__nsubplots = 1
1367 self.counter_imagwr = 0
1367 self.counter_imagwr = 0
1368 self.WIDTH = 800
1368 self.WIDTH = 800
1369 self.HEIGHT = 400
1369 self.HEIGHT = 400
1370 self.WIDTHPROF = 120
1370 self.WIDTHPROF = 120
1371 self.HEIGHTPROF = 0
1371 self.HEIGHTPROF = 0
1372 self.xdata = None
1372 self.xdata = None
1373 self.ydata = None
1373 self.ydata = None
1374
1374
1375 self.PLOT_CODE = BEACON_CODE
1375 self.PLOT_CODE = BEACON_CODE
1376
1376
1377 self.FTP_WEI = None
1377 self.FTP_WEI = None
1378 self.EXP_CODE = None
1378 self.EXP_CODE = None
1379 self.SUB_EXP_CODE = None
1379 self.SUB_EXP_CODE = None
1380 self.PLOT_POS = None
1380 self.PLOT_POS = None
1381
1381
1382 self.filename_phase = None
1382 self.filename_phase = None
1383
1383
1384 self.figfile = None
1384 self.figfile = None
1385
1385
1386 self.xmin = None
1386 self.xmin = None
1387 self.xmax = None
1387 self.xmax = None
1388
1388
1389 def getSubplots(self):
1389 def getSubplots(self):
1390
1390
1391 ncol = 1
1391 ncol = 1
1392 nrow = 1
1392 nrow = 1
1393
1393
1394 return nrow, ncol
1394 return nrow, ncol
1395
1395
1396 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1396 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1397
1397
1398 self.__showprofile = showprofile
1398 self.__showprofile = showprofile
1399 self.nplots = nplots
1399 self.nplots = nplots
1400
1400
1401 ncolspan = 7
1401 ncolspan = 7
1402 colspan = 6
1402 colspan = 6
1403 self.__nsubplots = 2
1403 self.__nsubplots = 2
1404
1404
1405 self.createFigure(id = id,
1405 self.createFigure(id = id,
1406 wintitle = wintitle,
1406 wintitle = wintitle,
1407 widthplot = self.WIDTH+self.WIDTHPROF,
1407 widthplot = self.WIDTH+self.WIDTHPROF,
1408 heightplot = self.HEIGHT+self.HEIGHTPROF,
1408 heightplot = self.HEIGHT+self.HEIGHTPROF,
1409 show=show)
1409 show=show)
1410
1410
1411 nrow, ncol = self.getSubplots()
1411 nrow, ncol = self.getSubplots()
1412
1412
1413 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1413 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1414
1414
1415 def save_phase(self, filename_phase):
1415 def save_phase(self, filename_phase):
1416 f = open(filename_phase,'w+')
1416 f = open(filename_phase,'w+')
1417 f.write('\n\n')
1417 f.write('\n\n')
1418 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
1418 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
1419 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
1419 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
1420 f.close()
1420 f.close()
1421
1421
1422 def save_data(self, filename_phase, data, data_datetime):
1422 def save_data(self, filename_phase, data, data_datetime):
1423 f=open(filename_phase,'a')
1423 f=open(filename_phase,'a')
1424 timetuple_data = data_datetime.timetuple()
1424 timetuple_data = data_datetime.timetuple()
1425 day = str(timetuple_data.tm_mday)
1425 day = str(timetuple_data.tm_mday)
1426 month = str(timetuple_data.tm_mon)
1426 month = str(timetuple_data.tm_mon)
1427 year = str(timetuple_data.tm_year)
1427 year = str(timetuple_data.tm_year)
1428 hour = str(timetuple_data.tm_hour)
1428 hour = str(timetuple_data.tm_hour)
1429 minute = str(timetuple_data.tm_min)
1429 minute = str(timetuple_data.tm_min)
1430 second = str(timetuple_data.tm_sec)
1430 second = str(timetuple_data.tm_sec)
1431 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
1431 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
1432 f.close()
1432 f.close()
1433
1433
1434
1434
1435 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
1435 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
1436 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
1436 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
1437 timerange=None,
1437 timerange=None,
1438 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1438 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1439 server=None, folder=None, username=None, password=None,
1439 server=None, folder=None, username=None, password=None,
1440 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1440 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1441
1441
1442 if dataOut.flagNoData:
1442 if dataOut.flagNoData:
1443 return dataOut
1443 return dataOut
1444
1444
1445 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
1445 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
1446 return
1446 return
1447
1447
1448 if pairsList == None:
1448 if pairsList == None:
1449 pairsIndexList = dataOut.pairsIndexList[:10]
1449 pairsIndexList = dataOut.pairsIndexList[:10]
1450 else:
1450 else:
1451 pairsIndexList = []
1451 pairsIndexList = []
1452 for pair in pairsList:
1452 for pair in pairsList:
1453 if pair not in dataOut.pairsList:
1453 if pair not in dataOut.pairsList:
1454 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
1454 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
1455 pairsIndexList.append(dataOut.pairsList.index(pair))
1455 pairsIndexList.append(dataOut.pairsList.index(pair))
1456
1456
1457 if pairsIndexList == []:
1457 if pairsIndexList == []:
1458 return
1458 return
1459
1459
1460 # if len(pairsIndexList) > 4:
1460 # if len(pairsIndexList) > 4:
1461 # pairsIndexList = pairsIndexList[0:4]
1461 # pairsIndexList = pairsIndexList[0:4]
1462
1462
1463 hmin_index = None
1463 hmin_index = None
1464 hmax_index = None
1464 hmax_index = None
1465
1465
1466 if hmin != None and hmax != None:
1466 if hmin != None and hmax != None:
1467 indexes = numpy.arange(dataOut.nHeights)
1467 indexes = numpy.arange(dataOut.nHeights)
1468 hmin_list = indexes[dataOut.heightList >= hmin]
1468 hmin_list = indexes[dataOut.heightList >= hmin]
1469 hmax_list = indexes[dataOut.heightList <= hmax]
1469 hmax_list = indexes[dataOut.heightList <= hmax]
1470
1470
1471 if hmin_list.any():
1471 if hmin_list.any():
1472 hmin_index = hmin_list[0]
1472 hmin_index = hmin_list[0]
1473
1473
1474 if hmax_list.any():
1474 if hmax_list.any():
1475 hmax_index = hmax_list[-1]+1
1475 hmax_index = hmax_list[-1]+1
1476
1476
1477 x = dataOut.getTimeRange()
1477 x = dataOut.getTimeRange()
1478 #y = dataOut.getHeiRange()
1478 #y = dataOut.getHeiRange()
1479
1479
1480
1480
1481 thisDatetime = dataOut.datatime
1481 thisDatetime = dataOut.datatime
1482
1482
1483 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1483 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1484 xlabel = "Local Time"
1484 xlabel = "Local Time"
1485 ylabel = "Phase (degrees)"
1485 ylabel = "Phase (degrees)"
1486
1486
1487 update_figfile = False
1487 update_figfile = False
1488
1488
1489 nplots = len(pairsIndexList)
1489 nplots = len(pairsIndexList)
1490 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
1490 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
1491 phase_beacon = numpy.zeros(len(pairsIndexList))
1491 phase_beacon = numpy.zeros(len(pairsIndexList))
1492 for i in range(nplots):
1492 for i in range(nplots):
1493 pair = dataOut.pairsList[pairsIndexList[i]]
1493 pair = dataOut.pairsList[pairsIndexList[i]]
1494 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
1494 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
1495 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
1495 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
1496 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
1496 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
1497 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
1497 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
1498 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
1498 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
1499
1499
1500 #print "Phase %d%d" %(pair[0], pair[1])
1500 #print "Phase %d%d" %(pair[0], pair[1])
1501 #print phase[dataOut.beacon_heiIndexList]
1501 #print phase[dataOut.beacon_heiIndexList]
1502
1502
1503 if dataOut.beacon_heiIndexList:
1503 if dataOut.beacon_heiIndexList:
1504 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
1504 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
1505 else:
1505 else:
1506 phase_beacon[i] = numpy.average(phase)
1506 phase_beacon[i] = numpy.average(phase)
1507
1507
1508 if not self.isConfig:
1508 if not self.isConfig:
1509
1509
1510 nplots = len(pairsIndexList)
1510 nplots = len(pairsIndexList)
1511
1511
1512 self.setup(id=id,
1512 self.setup(id=id,
1513 nplots=nplots,
1513 nplots=nplots,
1514 wintitle=wintitle,
1514 wintitle=wintitle,
1515 showprofile=showprofile,
1515 showprofile=showprofile,
1516 show=show)
1516 show=show)
1517
1517
1518 if timerange != None:
1518 if timerange != None:
1519 self.timerange = timerange
1519 self.timerange = timerange
1520
1520
1521 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1521 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1522
1522
1523 if ymin == None: ymin = 0
1523 if ymin == None: ymin = 0
1524 if ymax == None: ymax = 360
1524 if ymax == None: ymax = 360
1525
1525
1526 self.FTP_WEI = ftp_wei
1526 self.FTP_WEI = ftp_wei
1527 self.EXP_CODE = exp_code
1527 self.EXP_CODE = exp_code
1528 self.SUB_EXP_CODE = sub_exp_code
1528 self.SUB_EXP_CODE = sub_exp_code
1529 self.PLOT_POS = plot_pos
1529 self.PLOT_POS = plot_pos
1530
1530
1531 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1531 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1532 self.isConfig = True
1532 self.isConfig = True
1533 self.figfile = figfile
1533 self.figfile = figfile
1534 self.xdata = numpy.array([])
1534 self.xdata = numpy.array([])
1535 self.ydata = numpy.array([])
1535 self.ydata = numpy.array([])
1536
1536
1537 update_figfile = True
1537 update_figfile = True
1538
1538
1539 #open file beacon phase
1539 #open file beacon phase
1540 path = '%s%03d' %(self.PREFIX, self.id)
1540 path = '%s%03d' %(self.PREFIX, self.id)
1541 beacon_file = os.path.join(path,'%s.txt'%self.name)
1541 beacon_file = os.path.join(path,'%s.txt'%self.name)
1542 self.filename_phase = os.path.join(figpath,beacon_file)
1542 self.filename_phase = os.path.join(figpath,beacon_file)
1543 #self.save_phase(self.filename_phase)
1543 #self.save_phase(self.filename_phase)
1544
1544
1545
1545
1546 #store data beacon phase
1546 #store data beacon phase
1547 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
1547 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
1548
1548
1549 self.setWinTitle(title)
1549 self.setWinTitle(title)
1550
1550
1551
1551
1552 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1552 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1553
1553
1554 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
1554 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
1555
1555
1556 axes = self.axesList[0]
1556 axes = self.axesList[0]
1557
1557
1558 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1558 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1559
1559
1560 if len(self.ydata)==0:
1560 if len(self.ydata)==0:
1561 self.ydata = phase_beacon.reshape(-1,1)
1561 self.ydata = phase_beacon.reshape(-1,1)
1562 else:
1562 else:
1563 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
1563 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
1564
1564
1565
1565
1566 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1566 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1567 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1567 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1568 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1568 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1569 XAxisAsTime=True, grid='both'
1569 XAxisAsTime=True, grid='both'
1570 )
1570 )
1571
1571
1572 self.draw()
1572 self.draw()
1573
1573
1574 if dataOut.ltctime >= self.xmax:
1574 if dataOut.ltctime >= self.xmax:
1575 self.counter_imagwr = wr_period
1575 self.counter_imagwr = wr_period
1576 self.isConfig = False
1576 self.isConfig = False
1577 update_figfile = True
1577 update_figfile = True
1578
1578
1579 self.save(figpath=figpath,
1579 self.save(figpath=figpath,
1580 figfile=figfile,
1580 figfile=figfile,
1581 save=save,
1581 save=save,
1582 ftp=ftp,
1582 ftp=ftp,
1583 wr_period=wr_period,
1583 wr_period=wr_period,
1584 thisDatetime=thisDatetime,
1584 thisDatetime=thisDatetime,
1585 update_figfile=update_figfile)
1585 update_figfile=update_figfile)
1586
1586
1587 return dataOut #Yong No newline at end of file
1587 return dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,232 +1,232
1 '''
1 '''
2 Created on Jul 9, 2014
2 Created on Jul 9, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import datetime
7 import datetime
8 import numpy
8 import numpy
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator #YONG
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator #YONG
10 from schainpy.utils import log
10 from schainpy.utils import log
11 from .figure import Figure
11 from .figure import Figure
12
12
13
13
14 @MPDecorator
14 @MPDecorator
15 class Scope(Figure):
15 class Scope(Figure):
16
16
17 isConfig = None
17 isConfig = None
18
18
19 def __init__(self):#, **kwargs): #YONG
19 def __init__(self):#, **kwargs): #YONG
20 Figure.__init__(self)#, **kwargs)
20 Figure.__init__(self)#, **kwargs)
21 self.isConfig = False
21 self.isConfig = False
22 self.WIDTH = 300
22 self.WIDTH = 300
23 self.HEIGHT = 200
23 self.HEIGHT = 200
24 self.counter_imagwr = 0
24 self.counter_imagwr = 0
25
25
26 def getSubplots(self):
26 def getSubplots(self):
27
27
28 nrow = self.nplots
28 nrow = self.nplots
29 ncol = 3
29 ncol = 3
30 return nrow, ncol
30 return nrow, ncol
31
31
32 def setup(self, id, nplots, wintitle, show):
32 def setup(self, id, nplots, wintitle, show):
33
33
34 self.nplots = nplots
34 self.nplots = nplots
35
35
36 self.createFigure(id=id,
36 self.createFigure(id=id,
37 wintitle=wintitle,
37 wintitle=wintitle,
38 show=show)
38 show=show)
39
39
40 nrow,ncol = self.getSubplots()
40 nrow,ncol = self.getSubplots()
41 colspan = 3
41 colspan = 3
42 rowspan = 1
42 rowspan = 1
43
43
44 for i in range(nplots):
44 for i in range(nplots):
45 self.addAxes(nrow, ncol, i, 0, colspan, rowspan)
45 self.addAxes(nrow, ncol, i, 0, colspan, rowspan)
46
46
47 def plot_iq(self, x, y, id, channelIndexList, thisDatetime, wintitle, show, xmin, xmax, ymin, ymax):
47 def plot_iq(self, x, y, id, channelIndexList, thisDatetime, wintitle, show, xmin, xmax, ymin, ymax):
48 yreal = y[channelIndexList,:].real
48 yreal = y[channelIndexList,:].real
49 yimag = y[channelIndexList,:].imag
49 yimag = y[channelIndexList,:].imag
50
50
51 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
51 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
52 xlabel = "Range (Km)"
52 xlabel = "Range (Km)"
53 ylabel = "Intensity - IQ"
53 ylabel = "Intensity - IQ"
54
54
55 if not self.isConfig:
55 if not self.isConfig:
56 nplots = len(channelIndexList)
56 nplots = len(channelIndexList)
57
57
58 self.setup(id=id,
58 self.setup(id=id,
59 nplots=nplots,
59 nplots=nplots,
60 wintitle='',
60 wintitle='',
61 show=show)
61 show=show)
62
62
63 if xmin == None: xmin = numpy.nanmin(x)
63 if xmin == None: xmin = numpy.nanmin(x)
64 if xmax == None: xmax = numpy.nanmax(x)
64 if xmax == None: xmax = numpy.nanmax(x)
65 if ymin == None: ymin = min(numpy.nanmin(yreal),numpy.nanmin(yimag))
65 if ymin == None: ymin = min(numpy.nanmin(yreal),numpy.nanmin(yimag))
66 if ymax == None: ymax = max(numpy.nanmax(yreal),numpy.nanmax(yimag))
66 if ymax == None: ymax = max(numpy.nanmax(yreal),numpy.nanmax(yimag))
67
67
68 self.isConfig = True
68 self.isConfig = True
69
69
70 self.setWinTitle(title)
70 self.setWinTitle(title)
71
71
72 for i in range(len(self.axesList)):
72 for i in range(len(self.axesList)):
73 title = "Channel %d" %(i)
73 title = "Channel %d" %(i)
74 axes = self.axesList[i]
74 axes = self.axesList[i]
75
75
76 axes.pline(x, yreal[i,:],
76 axes.pline(x, yreal[i,:],
77 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
77 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
78 xlabel=xlabel, ylabel=ylabel, title=title)
78 xlabel=xlabel, ylabel=ylabel, title=title)
79
79
80 axes.addpline(x, yimag[i,:], idline=1, color="red", linestyle="solid", lw=2)
80 axes.addpline(x, yimag[i,:], idline=1, color="red", linestyle="solid", lw=2)
81
81
82 def plot_power(self, x, y, id, channelIndexList, thisDatetime, wintitle, show, xmin, xmax, ymin, ymax):
82 def plot_power(self, x, y, id, channelIndexList, thisDatetime, wintitle, show, xmin, xmax, ymin, ymax):
83 y = y[channelIndexList,:] * numpy.conjugate(y[channelIndexList,:])
83 y = y[channelIndexList,:] * numpy.conjugate(y[channelIndexList,:])
84 yreal = y.real
84 yreal = y.real
85
85
86 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
86 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
87 xlabel = "Range (Km)"
87 xlabel = "Range (Km)"
88 ylabel = "Intensity"
88 ylabel = "Intensity"
89
89
90 if not self.isConfig:
90 if not self.isConfig:
91 nplots = len(channelIndexList)
91 nplots = len(channelIndexList)
92
92
93 self.setup(id=id,
93 self.setup(id=id,
94 nplots=nplots,
94 nplots=nplots,
95 wintitle='',
95 wintitle='',
96 show=show)
96 show=show)
97
97
98 if xmin == None: xmin = numpy.nanmin(x)
98 if xmin == None: xmin = numpy.nanmin(x)
99 if xmax == None: xmax = numpy.nanmax(x)
99 if xmax == None: xmax = numpy.nanmax(x)
100 if ymin == None: ymin = numpy.nanmin(yreal)
100 if ymin == None: ymin = numpy.nanmin(yreal)
101 if ymax == None: ymax = numpy.nanmax(yreal)
101 if ymax == None: ymax = numpy.nanmax(yreal)
102
102
103 self.isConfig = True
103 self.isConfig = True
104
104
105 self.setWinTitle(title)
105 self.setWinTitle(title)
106
106
107 for i in range(len(self.axesList)):
107 for i in range(len(self.axesList)):
108 title = "Channel %d" %(i)
108 title = "Channel %d" %(i)
109 axes = self.axesList[i]
109 axes = self.axesList[i]
110 ychannel = yreal[i,:]
110 ychannel = yreal[i,:]
111 axes.pline(x, ychannel,
111 axes.pline(x, ychannel,
112 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
112 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
113 xlabel=xlabel, ylabel=ylabel, title=title)
113 xlabel=xlabel, ylabel=ylabel, title=title)
114
114
115
115
116 def run(self, dataOut, id, wintitle="", channelList=None,
116 def run(self, dataOut, id, wintitle="", channelList=None,
117 xmin=None, xmax=None, ymin=None, ymax=None, save=False,
117 xmin=None, xmax=None, ymin=None, ymax=None, save=False,
118 figpath='./', figfile=None, show=True, wr_period=1,
118 figpath='./', figfile=None, show=True, wr_period=1,
119 ftp=False, server=None, folder=None, username=None, password=None, type='power', **kwargs):
119 ftp=False, server=None, folder=None, username=None, password=None, type='power', **kwargs):
120
120
121 """
121 """
122
122
123 Input:
123 Input:
124 dataOut :
124 dataOut :
125 id :
125 id :
126 wintitle :
126 wintitle :
127 channelList :
127 channelList :
128 xmin : None,
128 xmin : None,
129 xmax : None,
129 xmax : None,
130 ymin : None,
130 ymin : None,
131 ymax : None,
131 ymax : None,
132 """
132 """
133 if dataOut.flagNoData:
133 if dataOut.flagNoData:
134 return dataOut
134 return dataOut
135
135
136 if channelList == None:
136 if channelList == None:
137 channelIndexList = dataOut.channelIndexList
137 channelIndexList = dataOut.channelIndexList
138 else:
138 else:
139 channelIndexList = []
139 channelIndexList = []
140 for channel in channelList:
140 for channel in channelList:
141 if channel not in dataOut.channelList:
141 if channel not in dataOut.channelList:
142 raise ValueError("Channel %d is not in dataOut.channelList")
142 raise ValueError("Channel %d is not in dataOut.channelList")
143 channelIndexList.append(dataOut.channelList.index(channel))
143 channelIndexList.append(dataOut.channelList.index(channel))
144
144
145 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
145 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
146
146
147 if dataOut.flagDataAsBlock:
147 if dataOut.flagDataAsBlock:
148
148
149 for i in range(dataOut.nProfiles):
149 for i in range(dataOut.nProfiles):
150
150
151 wintitle1 = wintitle + " [Profile = %d] " %i
151 wintitle1 = wintitle + " [Profile = %d] " %i
152
152
153 if type == "power":
153 if type == "power":
154 self.plot_power(dataOut.heightList,
154 self.plot_power(dataOut.heightList,
155 dataOut.data[:,i,:],
155 dataOut.data[:,i,:],
156 id,
156 id,
157 channelIndexList,
157 channelIndexList,
158 thisDatetime,
158 thisDatetime,
159 wintitle1,
159 wintitle1,
160 show,
160 show,
161 xmin,
161 xmin,
162 xmax,
162 xmax,
163 ymin,
163 ymin,
164 ymax)
164 ymax)
165
165
166 if type == "iq":
166 if type == "iq":
167 self.plot_iq(dataOut.heightList,
167 self.plot_iq(dataOut.heightList,
168 dataOut.data[:,i,:],
168 dataOut.data[:,i,:],
169 id,
169 id,
170 channelIndexList,
170 channelIndexList,
171 thisDatetime,
171 thisDatetime,
172 wintitle1,
172 wintitle1,
173 show,
173 show,
174 xmin,
174 xmin,
175 xmax,
175 xmax,
176 ymin,
176 ymin,
177 ymax)
177 ymax)
178
178
179 self.draw()
179 self.draw()
180
180
181 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
181 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
182 figfile = self.getFilename(name = str_datetime) + "_" + str(i)
182 figfile = self.getFilename(name = str_datetime) + "_" + str(i)
183
183
184 self.save(figpath=figpath,
184 self.save(figpath=figpath,
185 figfile=figfile,
185 figfile=figfile,
186 save=save,
186 save=save,
187 ftp=ftp,
187 ftp=ftp,
188 wr_period=wr_period,
188 wr_period=wr_period,
189 thisDatetime=thisDatetime)
189 thisDatetime=thisDatetime)
190
190
191 else:
191 else:
192 wintitle += " [Profile = %d] " %dataOut.profileIndex
192 wintitle += " [Profile = %d] " %dataOut.profileIndex
193
193
194 if type == "power":
194 if type == "power":
195 self.plot_power(dataOut.heightList,
195 self.plot_power(dataOut.heightList,
196 dataOut.data,
196 dataOut.data,
197 id,
197 id,
198 channelIndexList,
198 channelIndexList,
199 thisDatetime,
199 thisDatetime,
200 wintitle,
200 wintitle,
201 show,
201 show,
202 xmin,
202 xmin,
203 xmax,
203 xmax,
204 ymin,
204 ymin,
205 ymax)
205 ymax)
206
206
207 if type == "iq":
207 if type == "iq":
208 self.plot_iq(dataOut.heightList,
208 self.plot_iq(dataOut.heightList,
209 dataOut.data,
209 dataOut.data,
210 id,
210 id,
211 channelIndexList,
211 channelIndexList,
212 thisDatetime,
212 thisDatetime,
213 wintitle,
213 wintitle,
214 show,
214 show,
215 xmin,
215 xmin,
216 xmax,
216 xmax,
217 ymin,
217 ymin,
218 ymax)
218 ymax)
219
219
220 self.draw()
220 self.draw()
221
221
222 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") + "_" + str(dataOut.profileIndex)
222 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") + "_" + str(dataOut.profileIndex)
223 figfile = self.getFilename(name = str_datetime)
223 figfile = self.getFilename(name = str_datetime)
224
224
225 self.save(figpath=figpath,
225 self.save(figpath=figpath,
226 figfile=figfile,
226 figfile=figfile,
227 save=save,
227 save=save,
228 ftp=ftp,
228 ftp=ftp,
229 wr_period=wr_period,
229 wr_period=wr_period,
230 thisDatetime=thisDatetime)
230 thisDatetime=thisDatetime)
231
231
232 return dataOut No newline at end of file
232 return dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1826 +1,1828
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import inspect
12 import inspect
13 import time
13 import time
14 import datetime
14 import datetime
15 import traceback
15 import traceback
16 import zmq
16 import zmq
17
17
18 try:
18 try:
19 from gevent import sleep
19 from gevent import sleep
20 except:
20 except:
21 from time import sleep
21 from time import sleep
22
22
23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
25 from schainpy.utils import log
25 from schainpy.utils import log
26 import schainpy.admin
26 import schainpy.admin
27
27
28 LOCALTIME = True
28 LOCALTIME = True
29
29
30
30
31 def isNumber(cad):
31 def isNumber(cad):
32 """
32 """
33 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
33 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
34
34
35 Excepciones:
35 Excepciones:
36 Si un determinado string no puede ser convertido a numero
36 Si un determinado string no puede ser convertido a numero
37 Input:
37 Input:
38 str, string al cual se le analiza para determinar si convertible a un numero o no
38 str, string al cual se le analiza para determinar si convertible a un numero o no
39
39
40 Return:
40 Return:
41 True : si el string es uno numerico
41 True : si el string es uno numerico
42 False : no es un string numerico
42 False : no es un string numerico
43 """
43 """
44 try:
44 try:
45 float(cad)
45 float(cad)
46 return True
46 return True
47 except:
47 except:
48 return False
48 return False
49
49
50
50
51 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
51 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
52 """
52 """
53 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
53 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
54
54
55 Inputs:
55 Inputs:
56 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
56 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
57
57
58 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
58 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
59 segundos contados desde 01/01/1970.
59 segundos contados desde 01/01/1970.
60 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
60 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
61 segundos contados desde 01/01/1970.
61 segundos contados desde 01/01/1970.
62
62
63 Return:
63 Return:
64 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
64 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
65 fecha especificado, de lo contrario retorna False.
65 fecha especificado, de lo contrario retorna False.
66
66
67 Excepciones:
67 Excepciones:
68 Si el archivo no existe o no puede ser abierto
68 Si el archivo no existe o no puede ser abierto
69 Si la cabecera no puede ser leida.
69 Si la cabecera no puede ser leida.
70
70
71 """
71 """
72 basicHeaderObj = BasicHeader(LOCALTIME)
72 basicHeaderObj = BasicHeader(LOCALTIME)
73
73
74 try:
74 try:
75 fp = open(filename, 'rb')
75 fp = open(filename, 'rb')
76 except IOError:
76 except IOError:
77 print("The file %s can't be opened" % (filename))
77 print("The file %s can't be opened" % (filename))
78 return 0
78 return 0
79
79
80 sts = basicHeaderObj.read(fp)
80 sts = basicHeaderObj.read(fp)
81 fp.close()
81 fp.close()
82
82
83 if not(sts):
83 if not(sts):
84 print("Skipping the file %s because it has not a valid header" % (filename))
84 print("Skipping the file %s because it has not a valid header" % (filename))
85 return 0
85 return 0
86
86
87 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
87 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
88 return 0
88 return 0
89
89
90 return 1
90 return 1
91
91
92
92
93 def isTimeInRange(thisTime, startTime, endTime):
93 def isTimeInRange(thisTime, startTime, endTime):
94 if endTime >= startTime:
94 if endTime >= startTime:
95 if (thisTime < startTime) or (thisTime > endTime):
95 if (thisTime < startTime) or (thisTime > endTime):
96 return 0
96 return 0
97 return 1
97 return 1
98 else:
98 else:
99 if (thisTime < startTime) and (thisTime > endTime):
99 if (thisTime < startTime) and (thisTime > endTime):
100 return 0
100 return 0
101 return 1
101 return 1
102
102
103
103
104 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
104 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
105 """
105 """
106 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
106 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
107
107
108 Inputs:
108 Inputs:
109 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
109 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
110
110
111 startDate : fecha inicial del rango seleccionado en formato datetime.date
111 startDate : fecha inicial del rango seleccionado en formato datetime.date
112
112
113 endDate : fecha final del rango seleccionado en formato datetime.date
113 endDate : fecha final del rango seleccionado en formato datetime.date
114
114
115 startTime : tiempo inicial del rango seleccionado en formato datetime.time
115 startTime : tiempo inicial del rango seleccionado en formato datetime.time
116
116
117 endTime : tiempo final del rango seleccionado en formato datetime.time
117 endTime : tiempo final del rango seleccionado en formato datetime.time
118
118
119 Return:
119 Return:
120 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
120 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
121 fecha especificado, de lo contrario retorna False.
121 fecha especificado, de lo contrario retorna False.
122
122
123 Excepciones:
123 Excepciones:
124 Si el archivo no existe o no puede ser abierto
124 Si el archivo no existe o no puede ser abierto
125 Si la cabecera no puede ser leida.
125 Si la cabecera no puede ser leida.
126
126
127 """
127 """
128
128
129 try:
129 try:
130 fp = open(filename, 'rb')
130 fp = open(filename, 'rb')
131 except IOError:
131 except IOError:
132 print("The file %s can't be opened" % (filename))
132 print("The file %s can't be opened" % (filename))
133 return None
133 return None
134
134
135 firstBasicHeaderObj = BasicHeader(LOCALTIME)
135 firstBasicHeaderObj = BasicHeader(LOCALTIME)
136 systemHeaderObj = SystemHeader()
136 systemHeaderObj = SystemHeader()
137 radarControllerHeaderObj = RadarControllerHeader()
137 radarControllerHeaderObj = RadarControllerHeader()
138 processingHeaderObj = ProcessingHeader()
138 processingHeaderObj = ProcessingHeader()
139
139
140 lastBasicHeaderObj = BasicHeader(LOCALTIME)
140 lastBasicHeaderObj = BasicHeader(LOCALTIME)
141
141
142 sts = firstBasicHeaderObj.read(fp)
142 sts = firstBasicHeaderObj.read(fp)
143
143
144 if not(sts):
144 if not(sts):
145 print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
145 print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
146 return None
146 return None
147
147
148 if not systemHeaderObj.read(fp):
148 if not systemHeaderObj.read(fp):
149 return None
149 return None
150
150
151 if not radarControllerHeaderObj.read(fp):
151 if not radarControllerHeaderObj.read(fp):
152 return None
152 return None
153
153
154 if not processingHeaderObj.read(fp):
154 if not processingHeaderObj.read(fp):
155 return None
155 return None
156
156
157 filesize = os.path.getsize(filename)
157 filesize = os.path.getsize(filename)
158
158
159 offset = processingHeaderObj.blockSize + 24 # header size
159 offset = processingHeaderObj.blockSize + 24 # header size
160
160
161 if filesize <= offset:
161 if filesize <= offset:
162 print("[Reading] %s: This file has not enough data" % filename)
162 print("[Reading] %s: This file has not enough data" % filename)
163 return None
163 return None
164
164
165 fp.seek(-offset, 2)
165 fp.seek(-offset, 2)
166
166
167 sts = lastBasicHeaderObj.read(fp)
167 sts = lastBasicHeaderObj.read(fp)
168
168
169 fp.close()
169 fp.close()
170
170
171 thisDatetime = lastBasicHeaderObj.datatime
171 thisDatetime = lastBasicHeaderObj.datatime
172 thisTime_last_block = thisDatetime.time()
172 thisTime_last_block = thisDatetime.time()
173
173
174 thisDatetime = firstBasicHeaderObj.datatime
174 thisDatetime = firstBasicHeaderObj.datatime
175 thisDate = thisDatetime.date()
175 thisDate = thisDatetime.date()
176 thisTime_first_block = thisDatetime.time()
176 thisTime_first_block = thisDatetime.time()
177
177
178 # General case
178 # General case
179 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
179 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
180 #-----------o----------------------------o-----------
180 #-----------o----------------------------o-----------
181 # startTime endTime
181 # startTime endTime
182
182
183 if endTime >= startTime:
183 if endTime >= startTime:
184 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
184 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
185 return None
185 return None
186
186
187 return thisDatetime
187 return thisDatetime
188
188
189 # If endTime < startTime then endTime belongs to the next day
189 # If endTime < startTime then endTime belongs to the next day
190
190
191 #<<<<<<<<<<<o o>>>>>>>>>>>
191 #<<<<<<<<<<<o o>>>>>>>>>>>
192 #-----------o----------------------------o-----------
192 #-----------o----------------------------o-----------
193 # endTime startTime
193 # endTime startTime
194
194
195 if (thisDate == startDate) and (thisTime_last_block < startTime):
195 if (thisDate == startDate) and (thisTime_last_block < startTime):
196 return None
196 return None
197
197
198 if (thisDate == endDate) and (thisTime_first_block > endTime):
198 if (thisDate == endDate) and (thisTime_first_block > endTime):
199 return None
199 return None
200
200
201 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
201 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
202 return None
202 return None
203
203
204 return thisDatetime
204 return thisDatetime
205
205
206
206
207 def isFolderInDateRange(folder, startDate=None, endDate=None):
207 def isFolderInDateRange(folder, startDate=None, endDate=None):
208 """
208 """
209 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
209 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
210
210
211 Inputs:
211 Inputs:
212 folder : nombre completo del directorio.
212 folder : nombre completo del directorio.
213 Su formato deberia ser "/path_root/?YYYYDDD"
213 Su formato deberia ser "/path_root/?YYYYDDD"
214
214
215 siendo:
215 siendo:
216 YYYY : Anio (ejemplo 2015)
216 YYYY : Anio (ejemplo 2015)
217 DDD : Dia del anio (ejemplo 305)
217 DDD : Dia del anio (ejemplo 305)
218
218
219 startDate : fecha inicial del rango seleccionado en formato datetime.date
219 startDate : fecha inicial del rango seleccionado en formato datetime.date
220
220
221 endDate : fecha final del rango seleccionado en formato datetime.date
221 endDate : fecha final del rango seleccionado en formato datetime.date
222
222
223 Return:
223 Return:
224 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
224 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
225 fecha especificado, de lo contrario retorna False.
225 fecha especificado, de lo contrario retorna False.
226 Excepciones:
226 Excepciones:
227 Si el directorio no tiene el formato adecuado
227 Si el directorio no tiene el formato adecuado
228 """
228 """
229
229
230 basename = os.path.basename(folder)
230 basename = os.path.basename(folder)
231
231
232 if not isRadarFolder(basename):
232 if not isRadarFolder(basename):
233 print("The folder %s has not the rigth format" % folder)
233 print("The folder %s has not the rigth format" % folder)
234 return 0
234 return 0
235
235
236 if startDate and endDate:
236 if startDate and endDate:
237 thisDate = getDateFromRadarFolder(basename)
237 thisDate = getDateFromRadarFolder(basename)
238
238
239 if thisDate < startDate:
239 if thisDate < startDate:
240 return 0
240 return 0
241
241
242 if thisDate > endDate:
242 if thisDate > endDate:
243 return 0
243 return 0
244
244
245 return 1
245 return 1
246
246
247
247
248 def isFileInDateRange(filename, startDate=None, endDate=None):
248 def isFileInDateRange(filename, startDate=None, endDate=None):
249 """
249 """
250 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
250 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
251
251
252 Inputs:
252 Inputs:
253 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
253 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
254
254
255 Su formato deberia ser "?YYYYDDDsss"
255 Su formato deberia ser "?YYYYDDDsss"
256
256
257 siendo:
257 siendo:
258 YYYY : Anio (ejemplo 2015)
258 YYYY : Anio (ejemplo 2015)
259 DDD : Dia del anio (ejemplo 305)
259 DDD : Dia del anio (ejemplo 305)
260 sss : set
260 sss : set
261
261
262 startDate : fecha inicial del rango seleccionado en formato datetime.date
262 startDate : fecha inicial del rango seleccionado en formato datetime.date
263
263
264 endDate : fecha final del rango seleccionado en formato datetime.date
264 endDate : fecha final del rango seleccionado en formato datetime.date
265
265
266 Return:
266 Return:
267 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
267 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
268 fecha especificado, de lo contrario retorna False.
268 fecha especificado, de lo contrario retorna False.
269 Excepciones:
269 Excepciones:
270 Si el archivo no tiene el formato adecuado
270 Si el archivo no tiene el formato adecuado
271 """
271 """
272
272
273 basename = os.path.basename(filename)
273 basename = os.path.basename(filename)
274
274
275 if not isRadarFile(basename):
275 if not isRadarFile(basename):
276 print("The filename %s has not the rigth format" % filename)
276 print("The filename %s has not the rigth format" % filename)
277 return 0
277 return 0
278
278
279 if startDate and endDate:
279 if startDate and endDate:
280 thisDate = getDateFromRadarFile(basename)
280 thisDate = getDateFromRadarFile(basename)
281
281
282 if thisDate < startDate:
282 if thisDate < startDate:
283 return 0
283 return 0
284
284
285 if thisDate > endDate:
285 if thisDate > endDate:
286 return 0
286 return 0
287
287
288 return 1
288 return 1
289
289
290
290
291 def getFileFromSet(path, ext, set):
291 def getFileFromSet(path, ext, set):
292 validFilelist = []
292 validFilelist = []
293 fileList = os.listdir(path)
293 fileList = os.listdir(path)
294
294
295 # 0 1234 567 89A BCDE
295 # 0 1234 567 89A BCDE
296 # H YYYY DDD SSS .ext
296 # H YYYY DDD SSS .ext
297
297
298 for thisFile in fileList:
298 for thisFile in fileList:
299 try:
299 try:
300 year = int(thisFile[1:5])
300 year = int(thisFile[1:5])
301 doy = int(thisFile[5:8])
301 doy = int(thisFile[5:8])
302 except:
302 except:
303 continue
303 continue
304
304
305 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
305 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
306 continue
306 continue
307
307
308 validFilelist.append(thisFile)
308 validFilelist.append(thisFile)
309
309
310 myfile = fnmatch.filter(
310 myfile = fnmatch.filter(
311 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
311 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
312
312
313 if len(myfile) != 0:
313 if len(myfile) != 0:
314 return myfile[0]
314 return myfile[0]
315 else:
315 else:
316 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
316 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
317 print('the filename %s does not exist' % filename)
317 print('the filename %s does not exist' % filename)
318 print('...going to the last file: ')
318 print('...going to the last file: ')
319
319
320 if validFilelist:
320 if validFilelist:
321 validFilelist = sorted(validFilelist, key=str.lower)
321 validFilelist = sorted(validFilelist, key=str.lower)
322 return validFilelist[-1]
322 return validFilelist[-1]
323
323
324 return None
324 return None
325
325
326
326
327 def getlastFileFromPath(path, ext):
327 def getlastFileFromPath(path, ext):
328 """
328 """
329 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
329 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
330 al final de la depuracion devuelve el ultimo file de la lista que quedo.
330 al final de la depuracion devuelve el ultimo file de la lista que quedo.
331
331
332 Input:
332 Input:
333 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
333 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
334 ext : extension de los files contenidos en una carpeta
334 ext : extension de los files contenidos en una carpeta
335
335
336 Return:
336 Return:
337 El ultimo file de una determinada carpeta, no se considera el path.
337 El ultimo file de una determinada carpeta, no se considera el path.
338 """
338 """
339 validFilelist = []
339 validFilelist = []
340 fileList = os.listdir(path)
340 fileList = os.listdir(path)
341
341
342 # 0 1234 567 89A BCDE
342 # 0 1234 567 89A BCDE
343 # H YYYY DDD SSS .ext
343 # H YYYY DDD SSS .ext
344
344
345 for thisFile in fileList:
345 for thisFile in fileList:
346
346
347 year = thisFile[1:5]
347 year = thisFile[1:5]
348 if not isNumber(year):
348 if not isNumber(year):
349 continue
349 continue
350
350
351 doy = thisFile[5:8]
351 doy = thisFile[5:8]
352 if not isNumber(doy):
352 if not isNumber(doy):
353 continue
353 continue
354
354
355 year = int(year)
355 year = int(year)
356 doy = int(doy)
356 doy = int(doy)
357
357
358 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
358 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
359 continue
359 continue
360
360
361 validFilelist.append(thisFile)
361 validFilelist.append(thisFile)
362
362
363 if validFilelist:
363 if validFilelist:
364 validFilelist = sorted(validFilelist, key=str.lower)
364 validFilelist = sorted(validFilelist, key=str.lower)
365 return validFilelist[-1]
365 return validFilelist[-1]
366
366
367 return None
367 return None
368
368
369
369
370 def checkForRealPath(path, foldercounter, year, doy, set, ext):
370 def checkForRealPath(path, foldercounter, year, doy, set, ext):
371 """
371 """
372 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
372 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
373 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
373 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
374 el path exacto de un determinado file.
374 el path exacto de un determinado file.
375
375
376 Example :
376 Example :
377 nombre correcto del file es .../.../D2009307/P2009307367.ext
377 nombre correcto del file es .../.../D2009307/P2009307367.ext
378
378
379 Entonces la funcion prueba con las siguientes combinaciones
379 Entonces la funcion prueba con las siguientes combinaciones
380 .../.../y2009307367.ext
380 .../.../y2009307367.ext
381 .../.../Y2009307367.ext
381 .../.../Y2009307367.ext
382 .../.../x2009307/y2009307367.ext
382 .../.../x2009307/y2009307367.ext
383 .../.../x2009307/Y2009307367.ext
383 .../.../x2009307/Y2009307367.ext
384 .../.../X2009307/y2009307367.ext
384 .../.../X2009307/y2009307367.ext
385 .../.../X2009307/Y2009307367.ext
385 .../.../X2009307/Y2009307367.ext
386 siendo para este caso, la ultima combinacion de letras, identica al file buscado
386 siendo para este caso, la ultima combinacion de letras, identica al file buscado
387
387
388 Return:
388 Return:
389 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
389 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
390 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
390 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
391 para el filename
391 para el filename
392 """
392 """
393 fullfilename = None
393 fullfilename = None
394 find_flag = False
394 find_flag = False
395 filename = None
395 filename = None
396
396
397 prefixDirList = [None, 'd', 'D']
397 prefixDirList = [None, 'd', 'D']
398 if ext.lower() == ".r": # voltage
398 if ext.lower() == ".r": # voltage
399 prefixFileList = ['d', 'D']
399 prefixFileList = ['d', 'D']
400 elif ext.lower() == ".pdata": # spectra
400 elif ext.lower() == ".pdata": # spectra
401 prefixFileList = ['p', 'P']
401 prefixFileList = ['p', 'P']
402 else:
402 else:
403 return None, filename
403 return None, filename
404
404
405 # barrido por las combinaciones posibles
405 # barrido por las combinaciones posibles
406 for prefixDir in prefixDirList:
406 for prefixDir in prefixDirList:
407 thispath = path
407 thispath = path
408 if prefixDir != None:
408 if prefixDir != None:
409 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
409 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
410 if foldercounter == 0:
410 if foldercounter == 0:
411 thispath = os.path.join(path, "%s%04d%03d" %
411 thispath = os.path.join(path, "%s%04d%03d" %
412 (prefixDir, year, doy))
412 (prefixDir, year, doy))
413 else:
413 else:
414 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
414 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
415 prefixDir, year, doy, foldercounter))
415 prefixDir, year, doy, foldercounter))
416 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
416 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
417 # formo el nombre del file xYYYYDDDSSS.ext
417 # formo el nombre del file xYYYYDDDSSS.ext
418 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
418 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
419 fullfilename = os.path.join(
419 fullfilename = os.path.join(
420 thispath, filename) # formo el path completo
420 thispath, filename) # formo el path completo
421
421
422 if os.path.exists(fullfilename): # verifico que exista
422 if os.path.exists(fullfilename): # verifico que exista
423 find_flag = True
423 find_flag = True
424 break
424 break
425 if find_flag:
425 if find_flag:
426 break
426 break
427
427
428 if not(find_flag):
428 if not(find_flag):
429 return None, filename
429 return None, filename
430
430
431 return fullfilename, filename
431 return fullfilename, filename
432
432
433
433
434 def isRadarFolder(folder):
434 def isRadarFolder(folder):
435 try:
435 try:
436 year = int(folder[1:5])
436 year = int(folder[1:5])
437 doy = int(folder[5:8])
437 doy = int(folder[5:8])
438 except:
438 except:
439 return 0
439 return 0
440
440
441 return 1
441 return 1
442
442
443
443
444 def isRadarFile(file):
444 def isRadarFile(file):
445 try:
445 try:
446 year = int(file[1:5])
446 year = int(file[1:5])
447 doy = int(file[5:8])
447 doy = int(file[5:8])
448 set = int(file[8:11])
448 set = int(file[8:11])
449 except:
449 except:
450 return 0
450 return 0
451
451
452 return 1
452 return 1
453
453
454
454
455 def getDateFromRadarFile(file):
455 def getDateFromRadarFile(file):
456 try:
456 try:
457 year = int(file[1:5])
457 year = int(file[1:5])
458 doy = int(file[5:8])
458 doy = int(file[5:8])
459 set = int(file[8:11])
459 set = int(file[8:11])
460 except:
460 except:
461 return None
461 return None
462
462
463 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
463 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
464 return thisDate
464 return thisDate
465
465
466
466
467 def getDateFromRadarFolder(folder):
467 def getDateFromRadarFolder(folder):
468 try:
468 try:
469 year = int(folder[1:5])
469 year = int(folder[1:5])
470 doy = int(folder[5:8])
470 doy = int(folder[5:8])
471 except:
471 except:
472 return None
472 return None
473
473
474 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
474 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
475 return thisDate
475 return thisDate
476
476
477
477
478 class JRODataIO:
478 class JRODataIO:
479
479
480 c = 3E8
480 c = 3E8
481
481
482 isConfig = False
482 isConfig = False
483
483
484 basicHeaderObj = None
484 basicHeaderObj = None
485
485
486 systemHeaderObj = None
486 systemHeaderObj = None
487
487
488 radarControllerHeaderObj = None
488 radarControllerHeaderObj = None
489
489
490 processingHeaderObj = None
490 processingHeaderObj = None
491
491
492 dtype = None
492 dtype = None
493
493
494 pathList = []
494 pathList = []
495
495
496 filenameList = []
496 filenameList = []
497
497
498 filename = None
498 filename = None
499
499
500 ext = None
500 ext = None
501
501
502 flagIsNewFile = 1
502 flagIsNewFile = 1
503
503
504 flagDiscontinuousBlock = 0
504 flagDiscontinuousBlock = 0
505
505
506 flagIsNewBlock = 0
506 flagIsNewBlock = 0
507
507
508 fp = None
508 fp = None
509
509
510 firstHeaderSize = 0
510 firstHeaderSize = 0
511
511
512 basicHeaderSize = 24
512 basicHeaderSize = 24
513
513
514 versionFile = 1103
514 versionFile = 1103
515
515
516 fileSize = None
516 fileSize = None
517
517
518 # ippSeconds = None
518 # ippSeconds = None
519
519
520 fileSizeByHeader = None
520 fileSizeByHeader = None
521
521
522 fileIndex = None
522 fileIndex = None
523
523
524 profileIndex = None
524 profileIndex = None
525
525
526 blockIndex = None
526 blockIndex = None
527
527
528 nTotalBlocks = None
528 nTotalBlocks = None
529
529
530 maxTimeStep = 30
530 maxTimeStep = 30
531
531
532 lastUTTime = None
532 lastUTTime = None
533
533
534 datablock = None
534 datablock = None
535
535
536 dataOut = None
536 dataOut = None
537
537
538 blocksize = None
538 blocksize = None
539
539
540 getByBlock = False
540 getByBlock = False
541
541
542 def __init__(self):
542 def __init__(self):
543
543
544 raise NotImplementedError
544 raise NotImplementedError
545
545
546 def run(self):
546 def run(self):
547
547
548 raise NotImplementedError
548 raise NotImplementedError
549
549
550 def getDtypeWidth(self):
550 def getDtypeWidth(self):
551
551
552 dtype_index = get_dtype_index(self.dtype)
552 dtype_index = get_dtype_index(self.dtype)
553 dtype_width = get_dtype_width(dtype_index)
553 dtype_width = get_dtype_width(dtype_index)
554
554
555 return dtype_width
555 return dtype_width
556
556
557 def getAllowedArgs(self):
557 def getAllowedArgs(self):
558 if hasattr(self, '__attrs__'):
558 if hasattr(self, '__attrs__'):
559 return self.__attrs__
559 return self.__attrs__
560 else:
560 else:
561 return inspect.getargspec(self.run).args
561 return inspect.getargspec(self.run).args
562
562
563
563
564 class JRODataReader(JRODataIO):
564 class JRODataReader(JRODataIO):
565
565
566 online = 0
566 online = 0
567
567
568 realtime = 0
568 realtime = 0
569
569
570 nReadBlocks = 0
570 nReadBlocks = 0
571
571
572 delay = 10 # number of seconds waiting a new file
572 delay = 10 # number of seconds waiting a new file
573
573
574 nTries = 3 # quantity tries
574 nTries = 3 # quantity tries
575
575
576 nFiles = 3 # number of files for searching
576 nFiles = 3 # number of files for searching
577
577
578 path = None
578 path = None
579
579
580 foldercounter = 0
580 foldercounter = 0
581
581
582 flagNoMoreFiles = 0
582 flagNoMoreFiles = 0
583
583
584 datetimeList = []
584 datetimeList = []
585
585
586 __isFirstTimeOnline = 1
586 __isFirstTimeOnline = 1
587
587
588 __printInfo = True
588 __printInfo = True
589
589
590 profileIndex = None
590 profileIndex = None
591
591
592 nTxs = 1
592 nTxs = 1
593
593
594 txIndex = None
594 txIndex = None
595
595
596 # Added--------------------
596 # Added--------------------
597
597
598 selBlocksize = None
598 selBlocksize = None
599
599
600 selBlocktime = None
600 selBlocktime = None
601
601
602 def __init__(self):
602 def __init__(self):
603 """
603 """
604 This class is used to find data files
604 This class is used to find data files
605
605
606 Example:
606 Example:
607 reader = JRODataReader()
607 reader = JRODataReader()
608 fileList = reader.findDataFiles()
608 fileList = reader.findDataFiles()
609
609
610 """
610 """
611 pass
611 pass
612
612
613 def createObjByDefault(self):
613 def createObjByDefault(self):
614 """
614 """
615
615
616 """
616 """
617 raise NotImplementedError
617 raise NotImplementedError
618
618
619 def getBlockDimension(self):
619 def getBlockDimension(self):
620
620
621 raise NotImplementedError
621 raise NotImplementedError
622
622
623 def searchFilesOffLine(self,
623 def searchFilesOffLine(self,
624 path,
624 path,
625 startDate=None,
625 startDate=None,
626 endDate=None,
626 endDate=None,
627 startTime=datetime.time(0, 0, 0),
627 startTime=datetime.time(0, 0, 0),
628 endTime=datetime.time(23, 59, 59),
628 endTime=datetime.time(23, 59, 59),
629 set=None,
629 set=None,
630 expLabel='',
630 expLabel='',
631 ext='.r',
631 ext='.r',
632 cursor=None,
632 cursor=None,
633 skip=None,
633 skip=None,
634 walk=True):
634 walk=True):
635
635
636 self.filenameList = []
636 self.filenameList = []
637 self.datetimeList = []
637 self.datetimeList = []
638
638
639 pathList = []
639 pathList = []
640
640
641 dateList, pathList = self.findDatafiles(
641 dateList, pathList = self.findDatafiles(
642 path, startDate, endDate, expLabel, ext, walk, include_path=True)
642 path, startDate, endDate, expLabel, ext, walk, include_path=True)
643
643
644 if dateList == []:
644 if dateList == []:
645 return [], []
645 return [], []
646
646
647 if len(dateList) > 1:
647 if len(dateList) > 1:
648 print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)))
648 print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)))
649 else:
649 else:
650 print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]))
650 print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]))
651
651
652 filenameList = []
652 filenameList = []
653 datetimeList = []
653 datetimeList = []
654
654
655 for thisPath in pathList:
655 for thisPath in pathList:
656
656
657 fileList = glob.glob1(thisPath, "*%s" % ext)
657 fileList = glob.glob1(thisPath, "*%s" % ext)
658 fileList.sort()
658 fileList.sort()
659
659
660 for file in fileList:
660 for file in fileList:
661
661
662 filename = os.path.join(thisPath, file)
662 filename = os.path.join(thisPath, file)
663
663
664 if not isFileInDateRange(filename, startDate, endDate):
664 if not isFileInDateRange(filename, startDate, endDate):
665 continue
665 continue
666
666
667 thisDatetime = isFileInTimeRange(
667 thisDatetime = isFileInTimeRange(
668 filename, startDate, endDate, startTime, endTime)
668 filename, startDate, endDate, startTime, endTime)
669
669
670 if not(thisDatetime):
670 if not(thisDatetime):
671 continue
671 continue
672
672
673 filenameList.append(filename)
673 filenameList.append(filename)
674 datetimeList.append(thisDatetime)
674 datetimeList.append(thisDatetime)
675
675
676 if cursor is not None and skip is not None:
676 if cursor is not None and skip is not None:
677 filenameList = filenameList[cursor * skip:cursor * skip + skip]
677 filenameList = filenameList[cursor * skip:cursor * skip + skip]
678 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
678 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
679
679
680 if not(filenameList):
680 if not(filenameList):
681 print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path))
681 print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path))
682 return [], []
682 return [], []
683
683
684 print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime))
684 print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime))
685
685
686 # for i in range(len(filenameList)):
686 # for i in range(len(filenameList)):
687 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
687 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
688
688
689 self.filenameList = filenameList
689 self.filenameList = filenameList
690 self.datetimeList = datetimeList
690 self.datetimeList = datetimeList
691
691
692 return pathList, filenameList
692 return pathList, filenameList
693
693
694 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
694 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
695 """
695 """
696 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
696 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
697 devuelve el archivo encontrado ademas de otros datos.
697 devuelve el archivo encontrado ademas de otros datos.
698
698
699 Input:
699 Input:
700 path : carpeta donde estan contenidos los files que contiene data
700 path : carpeta donde estan contenidos los files que contiene data
701
701
702 expLabel : Nombre del subexperimento (subfolder)
702 expLabel : Nombre del subexperimento (subfolder)
703
703
704 ext : extension de los files
704 ext : extension de los files
705
705
706 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
706 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
707
707
708 Return:
708 Return:
709 directory : eL directorio donde esta el file encontrado
709 directory : eL directorio donde esta el file encontrado
710 filename : el ultimo file de una determinada carpeta
710 filename : el ultimo file de una determinada carpeta
711 year : el anho
711 year : el anho
712 doy : el numero de dia del anho
712 doy : el numero de dia del anho
713 set : el set del archivo
713 set : el set del archivo
714
714
715
715
716 """
716 """
717 if not os.path.isdir(path):
717 if not os.path.isdir(path):
718 return None, None, None, None, None, None
718 return None, None, None, None, None, None
719
719
720 dirList = []
720 dirList = []
721
721
722 if not walk:
722 if not walk:
723 fullpath = path
723 fullpath = path
724 foldercounter = 0
724 foldercounter = 0
725 else:
725 else:
726 # Filtra solo los directorios
726 # Filtra solo los directorios
727 for thisPath in os.listdir(path):
727 for thisPath in os.listdir(path):
728 if not os.path.isdir(os.path.join(path, thisPath)):
728 if not os.path.isdir(os.path.join(path, thisPath)):
729 continue
729 continue
730 if not isRadarFolder(thisPath):
730 if not isRadarFolder(thisPath):
731 continue
731 continue
732
732
733 dirList.append(thisPath)
733 dirList.append(thisPath)
734
734
735 if not(dirList):
735 if not(dirList):
736 return None, None, None, None, None, None
736 return None, None, None, None, None, None
737
737
738 dirList = sorted(dirList, key=str.lower)
738 dirList = sorted(dirList, key=str.lower)
739
739
740 doypath = dirList[-1]
740 doypath = dirList[-1]
741 foldercounter = int(doypath.split('_')[1]) if len(
741 foldercounter = int(doypath.split('_')[1]) if len(
742 doypath.split('_')) > 1 else 0
742 doypath.split('_')) > 1 else 0
743 fullpath = os.path.join(path, doypath, expLabel)
743 fullpath = os.path.join(path, doypath, expLabel)
744
744
745 print("[Reading] %s folder was found: " % (fullpath))
745 print("[Reading] %s folder was found: " % (fullpath))
746
746
747 if set == None:
747 if set == None:
748 filename = getlastFileFromPath(fullpath, ext)
748 filename = getlastFileFromPath(fullpath, ext)
749 else:
749 else:
750 filename = getFileFromSet(fullpath, ext, set)
750 filename = getFileFromSet(fullpath, ext, set)
751
751
752 if not(filename):
752 if not(filename):
753 return None, None, None, None, None, None
753 return None, None, None, None, None, None
754
754
755 print("[Reading] %s file was found" % (filename))
755 print("[Reading] %s file was found" % (filename))
756
756
757 if not(self.__verifyFile(os.path.join(fullpath, filename))):
757 if not(self.__verifyFile(os.path.join(fullpath, filename))):
758 return None, None, None, None, None, None
758 return None, None, None, None, None, None
759
759
760 year = int(filename[1:5])
760 year = int(filename[1:5])
761 doy = int(filename[5:8])
761 doy = int(filename[5:8])
762 set = int(filename[8:11])
762 set = int(filename[8:11])
763
763
764 return fullpath, foldercounter, filename, year, doy, set
764 return fullpath, foldercounter, filename, year, doy, set
765
765
766 def __setNextFileOffline(self):
766 def __setNextFileOffline(self):
767
767
768 idFile = self.fileIndex
768 idFile = self.fileIndex
769
769
770 while (True):
770 while (True):
771 idFile += 1
771 idFile += 1
772 if not(idFile < len(self.filenameList)):
772 if not(idFile < len(self.filenameList)):
773 self.flagNoMoreFiles = 1
773 self.flagNoMoreFiles = 1
774 # print "[Reading] No more Files"
774 # print "[Reading] No more Files"
775 return 0
775 return 0
776
776
777 filename = self.filenameList[idFile]
777 filename = self.filenameList[idFile]
778
778
779 if not(self.__verifyFile(filename)):
779 if not(self.__verifyFile(filename)):
780 continue
780 continue
781
781
782 fileSize = os.path.getsize(filename)
782 fileSize = os.path.getsize(filename)
783 fp = open(filename, 'rb')
783 fp = open(filename, 'rb')
784 break
784 break
785
785
786 self.flagIsNewFile = 1
786 self.flagIsNewFile = 1
787 self.fileIndex = idFile
787 self.fileIndex = idFile
788 self.filename = filename
788 self.filename = filename
789 self.fileSize = fileSize
789 self.fileSize = fileSize
790 self.fp = fp
790 self.fp = fp
791
791
792 # print "[Reading] Setting the file: %s"%self.filename
792 # print "[Reading] Setting the file: %s"%self.filename
793
793
794 return 1
794 return 1
795
795
796 def __setNextFileOnline(self):
796 def __setNextFileOnline(self):
797 """
797 """
798 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
798 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
799 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
799 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
800 siguientes.
800 siguientes.
801
801
802 Affected:
802 Affected:
803 self.flagIsNewFile
803 self.flagIsNewFile
804 self.filename
804 self.filename
805 self.fileSize
805 self.fileSize
806 self.fp
806 self.fp
807 self.set
807 self.set
808 self.flagNoMoreFiles
808 self.flagNoMoreFiles
809
809
810 Return:
810 Return:
811 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
811 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
812 1 : si el file fue abierto con exito y esta listo a ser leido
812 1 : si el file fue abierto con exito y esta listo a ser leido
813
813
814 Excepciones:
814 Excepciones:
815 Si un determinado file no puede ser abierto
815 Si un determinado file no puede ser abierto
816 """
816 """
817 nFiles = 0
817 nFiles = 0
818 fileOk_flag = False
818 fileOk_flag = False
819 firstTime_flag = True
819 firstTime_flag = True
820
820
821 self.set += 1
821 self.set += 1
822
822
823 if self.set > 999:
823 if self.set > 999:
824 self.set = 0
824 self.set = 0
825 self.foldercounter += 1
825 self.foldercounter += 1
826
826
827 # busca el 1er file disponible
827 # busca el 1er file disponible
828 fullfilename, filename = checkForRealPath(
828 fullfilename, filename = checkForRealPath(
829 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
829 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
830 if fullfilename:
830 if fullfilename:
831 if self.__verifyFile(fullfilename, False):
831 if self.__verifyFile(fullfilename, False):
832 fileOk_flag = True
832 fileOk_flag = True
833
833
834 # si no encuentra un file entonces espera y vuelve a buscar
834 # si no encuentra un file entonces espera y vuelve a buscar
835 if not(fileOk_flag):
835 if not(fileOk_flag):
836 # busco en los siguientes self.nFiles+1 files posibles
836 # busco en los siguientes self.nFiles+1 files posibles
837 for nFiles in range(self.nFiles + 1):
837 for nFiles in range(self.nFiles + 1):
838
838
839 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
839 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
840 tries = self.nTries
840 tries = self.nTries
841 else:
841 else:
842 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
842 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
843
843
844 for nTries in range(tries):
844 for nTries in range(tries):
845 if firstTime_flag:
845 if firstTime_flag:
846 print("\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1))
846 print("\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1))
847 sleep(self.delay)
847 sleep(self.delay)
848 else:
848 else:
849 print("\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext))
849 print("\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext))
850
850
851 fullfilename, filename = checkForRealPath(
851 fullfilename, filename = checkForRealPath(
852 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
852 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
853 if fullfilename:
853 if fullfilename:
854 if self.__verifyFile(fullfilename):
854 if self.__verifyFile(fullfilename):
855 fileOk_flag = True
855 fileOk_flag = True
856 break
856 break
857
857
858 if fileOk_flag:
858 if fileOk_flag:
859 break
859 break
860
860
861 firstTime_flag = False
861 firstTime_flag = False
862
862
863 log.warning('Skipping the file {} due to this file doesn\'t exist'.format(filename))
863 log.warning('Skipping the file {} due to this file doesn\'t exist'.format(filename))
864 self.set += 1
864 self.set += 1
865
865
866 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
866 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
867 if nFiles == (self.nFiles - 1):
867 if nFiles == (self.nFiles - 1):
868 self.set = 0
868 self.set = 0
869 self.doy += 1
869 self.doy += 1
870 self.foldercounter = 0
870 self.foldercounter = 0
871
871
872 if fileOk_flag:
872 if fileOk_flag:
873 self.fileSize = os.path.getsize(fullfilename)
873 self.fileSize = os.path.getsize(fullfilename)
874 self.filename = fullfilename
874 self.filename = fullfilename
875 self.flagIsNewFile = 1
875 self.flagIsNewFile = 1
876 if self.fp != None:
876 if self.fp != None:
877 self.fp.close()
877 self.fp.close()
878 self.fp = open(fullfilename, 'rb')
878 self.fp = open(fullfilename, 'rb')
879 self.flagNoMoreFiles = 0
879 self.flagNoMoreFiles = 0
880 # print '[Reading] Setting the file: %s' % fullfilename
880 # print '[Reading] Setting the file: %s' % fullfilename
881 else:
881 else:
882 self.fileSize = 0
882 self.fileSize = 0
883 self.filename = None
883 self.filename = None
884 self.flagIsNewFile = 0
884 self.flagIsNewFile = 0
885 self.fp = None
885 self.fp = None
886 self.flagNoMoreFiles = 1
886 self.flagNoMoreFiles = 1
887 # print '[Reading] No more files to read'
887 # print '[Reading] No more files to read'
888
888
889 return fileOk_flag
889 return fileOk_flag
890
890
891 def setNextFile(self):
891 def setNextFile(self):
892 if self.fp != None:
892 if self.fp != None:
893 self.fp.close()
893 self.fp.close()
894
894
895 if self.online:
895 if self.online:
896 newFile = self.__setNextFileOnline()
896 newFile = self.__setNextFileOnline()
897 else:
897 else:
898 newFile = self.__setNextFileOffline()
898 newFile = self.__setNextFileOffline()
899
899
900 if not(newFile):
900 if not(newFile):
901 self.dataOut.error = (-1, 'No more files to read')
901 self.dataOut.error = (-1, 'No more files to read')
902 return 0
902 return 0
903
903
904 if self.verbose:
904 if self.verbose:
905 print('[Reading] Setting the file: %s' % self.filename)
905 print('[Reading] Setting the file: %s' % self.filename)
906
906
907 self.__readFirstHeader()
907 self.__readFirstHeader()
908 self.nReadBlocks = 0
908 self.nReadBlocks = 0
909 return 1
909 return 1
910
910
911 def __waitNewBlock(self):
911 def __waitNewBlock(self):
912 """
912 """
913 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
913 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
914
914
915 Si el modo de lectura es OffLine siempre retorn 0
915 Si el modo de lectura es OffLine siempre retorn 0
916 """
916 """
917 if not self.online:
917 if not self.online:
918 return 0
918 return 0
919
919
920 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
920 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
921 return 0
921 return 0
922
922
923 currentPointer = self.fp.tell()
923 currentPointer = self.fp.tell()
924
924
925 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
925 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
926
926
927 for nTries in range(self.nTries):
927 for nTries in range(self.nTries):
928
928
929 self.fp.close()
929 self.fp.close()
930 self.fp = open(self.filename, 'rb')
930 self.fp = open(self.filename, 'rb')
931 self.fp.seek(currentPointer)
931 self.fp.seek(currentPointer)
932
932
933 self.fileSize = os.path.getsize(self.filename)
933 self.fileSize = os.path.getsize(self.filename)
934 currentSize = self.fileSize - currentPointer
934 currentSize = self.fileSize - currentPointer
935
935
936 if (currentSize >= neededSize):
936 if (currentSize >= neededSize):
937 self.basicHeaderObj.read(self.fp)
937 self.basicHeaderObj.read(self.fp)
938 return 1
938 return 1
939
939
940 if self.fileSize == self.fileSizeByHeader:
940 if self.fileSize == self.fileSizeByHeader:
941 # self.flagEoF = True
941 # self.flagEoF = True
942 return 0
942 return 0
943
943
944 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
944 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
945 sleep(self.delay)
945 sleep(self.delay)
946
946
947 return 0
947 return 0
948
948
949 def waitDataBlock(self, pointer_location):
949 def waitDataBlock(self, pointer_location):
950
950
951 currentPointer = pointer_location
951 currentPointer = pointer_location
952
952
953 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
953 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
954
954
955 for nTries in range(self.nTries):
955 for nTries in range(self.nTries):
956 self.fp.close()
956 self.fp.close()
957 self.fp = open(self.filename, 'rb')
957 self.fp = open(self.filename, 'rb')
958 self.fp.seek(currentPointer)
958 self.fp.seek(currentPointer)
959
959
960 self.fileSize = os.path.getsize(self.filename)
960 self.fileSize = os.path.getsize(self.filename)
961 currentSize = self.fileSize - currentPointer
961 currentSize = self.fileSize - currentPointer
962
962
963 if (currentSize >= neededSize):
963 if (currentSize >= neededSize):
964 return 1
964 return 1
965
965
966 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
966 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
967 sleep(self.delay)
967 sleep(self.delay)
968
968
969 return 0
969 return 0
970
970
971 def __jumpToLastBlock(self):
971 def __jumpToLastBlock(self):
972
972
973 if not(self.__isFirstTimeOnline):
973 if not(self.__isFirstTimeOnline):
974 return
974 return
975
975
976 csize = self.fileSize - self.fp.tell()
976 csize = self.fileSize - self.fp.tell()
977 blocksize = self.processingHeaderObj.blockSize
977 blocksize = self.processingHeaderObj.blockSize
978
978
979 # salta el primer bloque de datos
979 # salta el primer bloque de datos
980 if csize > self.processingHeaderObj.blockSize:
980 if csize > self.processingHeaderObj.blockSize:
981 self.fp.seek(self.fp.tell() + blocksize)
981 self.fp.seek(self.fp.tell() + blocksize)
982 else:
982 else:
983 return
983 return
984
984
985 csize = self.fileSize - self.fp.tell()
985 csize = self.fileSize - self.fp.tell()
986 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
986 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
987 while True:
987 while True:
988
988
989 if self.fp.tell() < self.fileSize:
989 if self.fp.tell() < self.fileSize:
990 self.fp.seek(self.fp.tell() + neededsize)
990 self.fp.seek(self.fp.tell() + neededsize)
991 else:
991 else:
992 self.fp.seek(self.fp.tell() - neededsize)
992 self.fp.seek(self.fp.tell() - neededsize)
993 break
993 break
994
994
995 # csize = self.fileSize - self.fp.tell()
995 # csize = self.fileSize - self.fp.tell()
996 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
996 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
997 # factor = int(csize/neededsize)
997 # factor = int(csize/neededsize)
998 # if factor > 0:
998 # if factor > 0:
999 # self.fp.seek(self.fp.tell() + factor*neededsize)
999 # self.fp.seek(self.fp.tell() + factor*neededsize)
1000
1000
1001 self.flagIsNewFile = 0
1001 self.flagIsNewFile = 0
1002 self.__isFirstTimeOnline = 0
1002 self.__isFirstTimeOnline = 0
1003
1003
1004 def __setNewBlock(self):
1004 def __setNewBlock(self):
1005 # if self.server is None:
1005 # if self.server is None:
1006 if self.fp == None:
1006 if self.fp == None:
1007 return 0
1007 return 0
1008
1008
1009 # if self.online:
1009 # if self.online:
1010 # self.__jumpToLastBlock()
1010 # self.__jumpToLastBlock()
1011
1011
1012 if self.flagIsNewFile:
1012 if self.flagIsNewFile:
1013 self.lastUTTime = self.basicHeaderObj.utc
1013 self.lastUTTime = self.basicHeaderObj.utc
1014 return 1
1014 return 1
1015
1015
1016 if self.realtime:
1016 if self.realtime:
1017 self.flagDiscontinuousBlock = 1
1017 self.flagDiscontinuousBlock = 1
1018 if not(self.setNextFile()):
1018 if not(self.setNextFile()):
1019 return 0
1019 return 0
1020 else:
1020 else:
1021 return 1
1021 return 1
1022 # if self.server is None:
1022 # if self.server is None:
1023 currentSize = self.fileSize - self.fp.tell()
1023 currentSize = self.fileSize - self.fp.tell()
1024 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1024 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1025 if (currentSize >= neededSize):
1025 if (currentSize >= neededSize):
1026 self.basicHeaderObj.read(self.fp)
1026 self.basicHeaderObj.read(self.fp)
1027 self.lastUTTime = self.basicHeaderObj.utc
1027 self.lastUTTime = self.basicHeaderObj.utc
1028 return 1
1028 return 1
1029 # else:
1029 # else:
1030 # self.basicHeaderObj.read(self.zHeader)
1030 # self.basicHeaderObj.read(self.zHeader)
1031 # self.lastUTTime = self.basicHeaderObj.utc
1031 # self.lastUTTime = self.basicHeaderObj.utc
1032 # return 1
1032 # return 1
1033 if self.__waitNewBlock():
1033 if self.__waitNewBlock():
1034 self.lastUTTime = self.basicHeaderObj.utc
1034 self.lastUTTime = self.basicHeaderObj.utc
1035 return 1
1035 return 1
1036 # if self.server is None:
1036 # if self.server is None:
1037 if not(self.setNextFile()):
1037 if not(self.setNextFile()):
1038 return 0
1038 return 0
1039
1039
1040 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1040 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1041 self.lastUTTime = self.basicHeaderObj.utc
1041 self.lastUTTime = self.basicHeaderObj.utc
1042
1042
1043 self.flagDiscontinuousBlock = 0
1043 self.flagDiscontinuousBlock = 0
1044
1044
1045 if deltaTime > self.maxTimeStep:
1045 if deltaTime > self.maxTimeStep:
1046 self.flagDiscontinuousBlock = 1
1046 self.flagDiscontinuousBlock = 1
1047
1047
1048 return 1
1048 return 1
1049
1049
1050 def readNextBlock(self):
1050 def readNextBlock(self):
1051
1051
1052 # Skip block out of startTime and endTime
1052 # Skip block out of startTime and endTime
1053 while True:
1053 while True:
1054 if not(self.__setNewBlock()):
1054 if not(self.__setNewBlock()):
1055 self.dataOut.error = (-1, 'No more files to read')
1055 self.dataOut.error = (-1, 'No more files to read')
1056 return 0
1056 return 0
1057
1057
1058 if not(self.readBlock()):
1058 if not(self.readBlock()):
1059 return 0
1059 return 0
1060
1060
1061 self.getBasicHeader()
1061 self.getBasicHeader()
1062 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1062 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1063 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1063 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1064 self.processingHeaderObj.dataBlocksPerFile,
1064 self.processingHeaderObj.dataBlocksPerFile,
1065 self.dataOut.datatime.ctime()))
1065 self.dataOut.datatime.ctime()))
1066 continue
1066 continue
1067
1067
1068 break
1068 break
1069
1069
1070 if self.verbose:
1070 if self.verbose:
1071 print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1071 print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1072 self.processingHeaderObj.dataBlocksPerFile,
1072 self.processingHeaderObj.dataBlocksPerFile,
1073 self.dataOut.datatime.ctime()))
1073 self.dataOut.datatime.ctime()))
1074 return 1
1074 return 1
1075
1075
1076 def __readFirstHeader(self):
1076 def __readFirstHeader(self):
1077
1077
1078 self.basicHeaderObj.read(self.fp)
1078 self.basicHeaderObj.read(self.fp)
1079 self.systemHeaderObj.read(self.fp)
1079 self.systemHeaderObj.read(self.fp)
1080 self.radarControllerHeaderObj.read(self.fp)
1080 self.radarControllerHeaderObj.read(self.fp)
1081 self.processingHeaderObj.read(self.fp)
1081 self.processingHeaderObj.read(self.fp)
1082
1082
1083 self.firstHeaderSize = self.basicHeaderObj.size
1083 self.firstHeaderSize = self.basicHeaderObj.size
1084
1084
1085 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1085 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1086 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1086 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1087 if datatype == 0:
1087 if datatype == 0:
1088 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1088 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1089 elif datatype == 1:
1089 elif datatype == 1:
1090 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1090 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1091 elif datatype == 2:
1091 elif datatype == 2:
1092 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1092 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1093 elif datatype == 3:
1093 elif datatype == 3:
1094 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1094 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1095 elif datatype == 4:
1095 elif datatype == 4:
1096 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1096 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1097 elif datatype == 5:
1097 elif datatype == 5:
1098 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1098 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1099 else:
1099 else:
1100 raise ValueError('Data type was not defined')
1100 raise ValueError('Data type was not defined')
1101
1101
1102 self.dtype = datatype_str
1102 self.dtype = datatype_str
1103 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1103 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1104 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1104 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1105 self.firstHeaderSize + self.basicHeaderSize * \
1105 self.firstHeaderSize + self.basicHeaderSize * \
1106 (self.processingHeaderObj.dataBlocksPerFile - 1)
1106 (self.processingHeaderObj.dataBlocksPerFile - 1)
1107 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1107 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1108 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1108 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1109 self.getBlockDimension()
1109 self.getBlockDimension()
1110
1110
1111 def __verifyFile(self, filename, msgFlag=True):
1111 def __verifyFile(self, filename, msgFlag=True):
1112
1112
1113 msg = None
1113 msg = None
1114
1114
1115 try:
1115 try:
1116 fp = open(filename, 'rb')
1116 fp = open(filename, 'rb')
1117 except IOError:
1117 except IOError:
1118
1118
1119 if msgFlag:
1119 if msgFlag:
1120 print("[Reading] File %s can't be opened" % (filename))
1120 print("[Reading] File %s can't be opened" % (filename))
1121
1121
1122 return False
1122 return False
1123
1123
1124 currentPosition = fp.tell()
1124 currentPosition = fp.tell()
1125 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1125 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1126
1126
1127 if neededSize == 0:
1127 if neededSize == 0:
1128 basicHeaderObj = BasicHeader(LOCALTIME)
1128 basicHeaderObj = BasicHeader(LOCALTIME)
1129 systemHeaderObj = SystemHeader()
1129 systemHeaderObj = SystemHeader()
1130 radarControllerHeaderObj = RadarControllerHeader()
1130 radarControllerHeaderObj = RadarControllerHeader()
1131 processingHeaderObj = ProcessingHeader()
1131 processingHeaderObj = ProcessingHeader()
1132
1132
1133 if not(basicHeaderObj.read(fp)):
1133 if not(basicHeaderObj.read(fp)):
1134 fp.close()
1134 fp.close()
1135 return False
1135 return False
1136
1136
1137 if not(systemHeaderObj.read(fp)):
1137 if not(systemHeaderObj.read(fp)):
1138 fp.close()
1138 fp.close()
1139 return False
1139 return False
1140
1140
1141 if not(radarControllerHeaderObj.read(fp)):
1141 if not(radarControllerHeaderObj.read(fp)):
1142 fp.close()
1142 fp.close()
1143 return False
1143 return False
1144
1144
1145 if not(processingHeaderObj.read(fp)):
1145 if not(processingHeaderObj.read(fp)):
1146 fp.close()
1146 fp.close()
1147 return False
1147 return False
1148
1148
1149 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1149 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1150 else:
1150 else:
1151 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1151 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1152
1152
1153 fp.close()
1153 fp.close()
1154
1154
1155 fileSize = os.path.getsize(filename)
1155 fileSize = os.path.getsize(filename)
1156 currentSize = fileSize - currentPosition
1156 currentSize = fileSize - currentPosition
1157
1157
1158 if currentSize < neededSize:
1158 if currentSize < neededSize:
1159 if msgFlag and (msg != None):
1159 if msgFlag and (msg != None):
1160 print(msg)
1160 print(msg)
1161 return False
1161 return False
1162
1162
1163 return True
1163 return True
1164
1164
1165 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1165 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1166
1166
1167 path_empty = True
1167 path_empty = True
1168
1168
1169 dateList = []
1169 dateList = []
1170 pathList = []
1170 pathList = []
1171
1171
1172 multi_path = path.split(',')
1172 multi_path = path.split(',')
1173
1173
1174 if not walk:
1174 if not walk:
1175
1175
1176 for single_path in multi_path:
1176 for single_path in multi_path:
1177
1177
1178 if not os.path.isdir(single_path):
1178 if not os.path.isdir(single_path):
1179 continue
1179 continue
1180
1180
1181 fileList = glob.glob1(single_path, "*" + ext)
1181 fileList = glob.glob1(single_path, "*" + ext)
1182
1182
1183 if not fileList:
1183 if not fileList:
1184 continue
1184 continue
1185
1185
1186 path_empty = False
1186 path_empty = False
1187
1187
1188 fileList.sort()
1188 fileList.sort()
1189
1189
1190 for thisFile in fileList:
1190 for thisFile in fileList:
1191
1191
1192 if not os.path.isfile(os.path.join(single_path, thisFile)):
1192 if not os.path.isfile(os.path.join(single_path, thisFile)):
1193 continue
1193 continue
1194
1194
1195 if not isRadarFile(thisFile):
1195 if not isRadarFile(thisFile):
1196 continue
1196 continue
1197
1197
1198 if not isFileInDateRange(thisFile, startDate, endDate):
1198 if not isFileInDateRange(thisFile, startDate, endDate):
1199 continue
1199 continue
1200
1200
1201 thisDate = getDateFromRadarFile(thisFile)
1201 thisDate = getDateFromRadarFile(thisFile)
1202
1202
1203 if thisDate in dateList:
1203 if thisDate in dateList:
1204 continue
1204 continue
1205
1205
1206 dateList.append(thisDate)
1206 dateList.append(thisDate)
1207 pathList.append(single_path)
1207 pathList.append(single_path)
1208
1208
1209 else:
1209 else:
1210 for single_path in multi_path:
1210 for single_path in multi_path:
1211
1211
1212 if not os.path.isdir(single_path):
1212 if not os.path.isdir(single_path):
1213 continue
1213 continue
1214
1214
1215 dirList = []
1215 dirList = []
1216
1216
1217 for thisPath in os.listdir(single_path):
1217 for thisPath in os.listdir(single_path):
1218
1218
1219 if not os.path.isdir(os.path.join(single_path, thisPath)):
1219 if not os.path.isdir(os.path.join(single_path, thisPath)):
1220 continue
1220 continue
1221
1221
1222 if not isRadarFolder(thisPath):
1222 if not isRadarFolder(thisPath):
1223 continue
1223 continue
1224
1224
1225 if not isFolderInDateRange(thisPath, startDate, endDate):
1225 if not isFolderInDateRange(thisPath, startDate, endDate):
1226 continue
1226 continue
1227
1227
1228 dirList.append(thisPath)
1228 dirList.append(thisPath)
1229
1229
1230 if not dirList:
1230 if not dirList:
1231 continue
1231 continue
1232
1232
1233 dirList.sort()
1233 dirList.sort()
1234
1234
1235 for thisDir in dirList:
1235 for thisDir in dirList:
1236
1236
1237 datapath = os.path.join(single_path, thisDir, expLabel)
1237 datapath = os.path.join(single_path, thisDir, expLabel)
1238 fileList = glob.glob1(datapath, "*" + ext)
1238 fileList = glob.glob1(datapath, "*" + ext)
1239
1239
1240 if not fileList:
1240 if not fileList:
1241 continue
1241 continue
1242
1242
1243 path_empty = False
1243 path_empty = False
1244
1244
1245 thisDate = getDateFromRadarFolder(thisDir)
1245 thisDate = getDateFromRadarFolder(thisDir)
1246
1246
1247 pathList.append(datapath)
1247 pathList.append(datapath)
1248 dateList.append(thisDate)
1248 dateList.append(thisDate)
1249
1249
1250 dateList.sort()
1250 dateList.sort()
1251
1251
1252 if walk:
1252 if walk:
1253 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1253 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1254 else:
1254 else:
1255 pattern_path = multi_path[0]
1255 pattern_path = multi_path[0]
1256
1256
1257 if path_empty:
1257 if path_empty:
1258 print("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
1258 print("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
1259 else:
1259 else:
1260 if not dateList:
1260 if not dateList:
1261 print("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
1261 print("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
1262
1262
1263 if include_path:
1263 if include_path:
1264 return dateList, pathList
1264 return dateList, pathList
1265
1265
1266 return dateList
1266 return dateList
1267
1267
1268 def setup(self,
1268 def setup(self,
1269 path=None,
1269 path=None,
1270 startDate=None,
1270 startDate=None,
1271 endDate=None,
1271 endDate=None,
1272 startTime=datetime.time(0, 0, 0),
1272 startTime=datetime.time(0, 0, 0),
1273 endTime=datetime.time(23, 59, 59),
1273 endTime=datetime.time(23, 59, 59),
1274 set=None,
1274 set=None,
1275 expLabel="",
1275 expLabel="",
1276 ext=None,
1276 ext=None,
1277 online=False,
1277 online=False,
1278 delay=60,
1278 delay=60,
1279 walk=True,
1279 walk=True,
1280 getblock=False,
1280 getblock=False,
1281 nTxs=1,
1281 nTxs=1,
1282 realtime=False,
1282 realtime=False,
1283 blocksize=None,
1283 blocksize=None,
1284 blocktime=None,
1284 blocktime=None,
1285 skip=None,
1285 skip=None,
1286 cursor=None,
1286 cursor=None,
1287 warnings=True,
1287 warnings=True,
1288 verbose=True,
1288 verbose=True,
1289 server=None,
1289 server=None,
1290 format=None,
1290 format=None,
1291 oneDDict=None,
1291 oneDDict=None,
1292 twoDDict=None,
1292 twoDDict=None,
1293 ind2DList=None):
1293 ind2DList=None):
1294 if server is not None:
1294 if server is not None:
1295 if 'tcp://' in server:
1295 if 'tcp://' in server:
1296 address = server
1296 address = server
1297 else:
1297 else:
1298 address = 'ipc:///tmp/%s' % server
1298 address = 'ipc:///tmp/%s' % server
1299 self.server = address
1299 self.server = address
1300 self.context = zmq.Context()
1300 self.context = zmq.Context()
1301 self.receiver = self.context.socket(zmq.PULL)
1301 self.receiver = self.context.socket(zmq.PULL)
1302 self.receiver.connect(self.server)
1302 self.receiver.connect(self.server)
1303 time.sleep(0.5)
1303 time.sleep(0.5)
1304 print('[Starting] ReceiverData from {}'.format(self.server))
1304 print('[Starting] ReceiverData from {}'.format(self.server))
1305 else:
1305 else:
1306 self.server = None
1306 self.server = None
1307 if path == None:
1307 if path == None:
1308 raise ValueError("[Reading] The path is not valid")
1308 raise ValueError("[Reading] The path is not valid")
1309
1309
1310 if ext == None:
1310 if ext == None:
1311 ext = self.ext
1311 ext = self.ext
1312
1312
1313 if online:
1313 if online:
1314 print("[Reading] Searching files in online mode...")
1314 print("[Reading] Searching files in online mode...")
1315
1315
1316 for nTries in range(self.nTries):
1316 for nTries in range(self.nTries):
1317 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1317 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1318 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1318 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1319
1319
1320 if fullpath:
1320 if fullpath:
1321 break
1321 break
1322
1322
1323 print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1))
1323 print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1))
1324 sleep(self.delay)
1324 sleep(self.delay)
1325
1325
1326 if not(fullpath):
1326 if not(fullpath):
1327 self.dataOut.error = (-1, 'There isn\'t any valid file in {}'.format(path))
1327 self.dataOut.error = (-1, 'There isn\'t any valid file in {}'.format(path))
1328 return
1328 return
1329
1329
1330 self.year = year
1330 self.year = year
1331 self.doy = doy
1331 self.doy = doy
1332 self.set = set - 1
1332 self.set = set - 1
1333 self.path = path
1333 self.path = path
1334 self.foldercounter = foldercounter
1334 self.foldercounter = foldercounter
1335 last_set = None
1335 last_set = None
1336 else:
1336 else:
1337 print("[Reading] Searching files in offline mode ...")
1337 print("[Reading] Searching files in offline mode ...")
1338 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1338 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1339 startTime=startTime, endTime=endTime,
1339 startTime=startTime, endTime=endTime,
1340 set=set, expLabel=expLabel, ext=ext,
1340 set=set, expLabel=expLabel, ext=ext,
1341 walk=walk, cursor=cursor,
1341 walk=walk, cursor=cursor,
1342 skip=skip)
1342 skip=skip)
1343
1343
1344 if not(pathList):
1344 if not(pathList):
1345 self.fileIndex = -1
1345 self.fileIndex = -1
1346 self.pathList = []
1346 self.pathList = []
1347 self.filenameList = []
1347 self.filenameList = []
1348 return
1348 return
1349
1349
1350 self.fileIndex = -1
1350 self.fileIndex = -1
1351 self.pathList = pathList
1351 self.pathList = pathList
1352 self.filenameList = filenameList
1352 self.filenameList = filenameList
1353 file_name = os.path.basename(filenameList[-1])
1353 file_name = os.path.basename(filenameList[-1])
1354 basename, ext = os.path.splitext(file_name)
1354 basename, ext = os.path.splitext(file_name)
1355 last_set = int(basename[-3:])
1355 last_set = int(basename[-3:])
1356
1356
1357 self.online = online
1357 self.online = online
1358 self.realtime = realtime
1358 self.realtime = realtime
1359 self.delay = delay
1359 self.delay = delay
1360 ext = ext.lower()
1360 ext = ext.lower()
1361 self.ext = ext
1361 self.ext = ext
1362 self.getByBlock = getblock
1362 self.getByBlock = getblock
1363 self.nTxs = nTxs
1363 self.nTxs = nTxs
1364 self.startTime = startTime
1364 self.startTime = startTime
1365 self.endTime = endTime
1365 self.endTime = endTime
1366 self.endDate = endDate
1366 self.endDate = endDate
1367 self.startDate = startDate
1367 self.startDate = startDate
1368 # Added-----------------
1368 # Added-----------------
1369 self.selBlocksize = blocksize
1369 self.selBlocksize = blocksize
1370 self.selBlocktime = blocktime
1370 self.selBlocktime = blocktime
1371
1371
1372 # Verbose-----------
1372 # Verbose-----------
1373 self.verbose = verbose
1373 self.verbose = verbose
1374 self.warnings = warnings
1374 self.warnings = warnings
1375
1375
1376 if not(self.setNextFile()):
1376 if not(self.setNextFile()):
1377 if (startDate != None) and (endDate != None):
1377 if (startDate != None) and (endDate != None):
1378 print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
1378 print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
1379 elif startDate != None:
1379 elif startDate != None:
1380 print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
1380 print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
1381 else:
1381 else:
1382 print("[Reading] No files")
1382 print("[Reading] No files")
1383
1383
1384 self.fileIndex = -1
1384 self.fileIndex = -1
1385 self.pathList = []
1385 self.pathList = []
1386 self.filenameList = []
1386 self.filenameList = []
1387 return
1387 return
1388
1388
1389 # self.getBasicHeader()
1389 # self.getBasicHeader()
1390
1390
1391 if last_set != None:
1391 if last_set != None:
1392 self.dataOut.last_block = last_set * \
1392 self.dataOut.last_block = last_set * \
1393 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1393 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1394 return
1394 return
1395
1395
1396 def getBasicHeader(self):
1396 def getBasicHeader(self):
1397
1397
1398 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1398 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1399 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1399 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1400
1400
1401 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1401 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1402
1402
1403 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1403 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1404
1404
1405 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1405 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1406
1406
1407 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1407 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1408
1408
1409 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1409 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1410
1410
1411 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1411 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1412
1412
1413 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1413 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1414
1414
1415 def getFirstHeader(self):
1415 def getFirstHeader(self):
1416
1416
1417 raise NotImplementedError
1417 raise NotImplementedError
1418
1418
1419 def getData(self):
1419 def getData(self):
1420
1420
1421 raise NotImplementedError
1421 raise NotImplementedError
1422
1422
1423 def hasNotDataInBuffer(self):
1423 def hasNotDataInBuffer(self):
1424
1424
1425 raise NotImplementedError
1425 raise NotImplementedError
1426
1426
1427 def readBlock(self):
1427 def readBlock(self):
1428
1428
1429 raise NotImplementedError
1429 raise NotImplementedError
1430
1430
1431 def isEndProcess(self):
1431 def isEndProcess(self):
1432
1432
1433 return self.flagNoMoreFiles
1433 return self.flagNoMoreFiles
1434
1434
1435 def printReadBlocks(self):
1435 def printReadBlocks(self):
1436
1436
1437 print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
1437 print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
1438
1438
1439 def printTotalBlocks(self):
1439 def printTotalBlocks(self):
1440
1440
1441 print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
1441 print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
1442
1442
1443 def printNumberOfBlock(self):
1443 def printNumberOfBlock(self):
1444 'SPAM!'
1444 'SPAM!'
1445
1445
1446 # if self.flagIsNewBlock:
1446 # if self.flagIsNewBlock:
1447 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1447 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1448 # self.processingHeaderObj.dataBlocksPerFile,
1448 # self.processingHeaderObj.dataBlocksPerFile,
1449 # self.dataOut.datatime.ctime())
1449 # self.dataOut.datatime.ctime())
1450
1450
1451 def printInfo(self):
1451 def printInfo(self):
1452
1452
1453 if self.__printInfo == False:
1453 if self.__printInfo == False:
1454 return
1454 return
1455
1455
1456 self.basicHeaderObj.printInfo()
1456 self.basicHeaderObj.printInfo()
1457 self.systemHeaderObj.printInfo()
1457 self.systemHeaderObj.printInfo()
1458 self.radarControllerHeaderObj.printInfo()
1458 self.radarControllerHeaderObj.printInfo()
1459 self.processingHeaderObj.printInfo()
1459 self.processingHeaderObj.printInfo()
1460
1460
1461 self.__printInfo = False
1461 self.__printInfo = False
1462
1462
1463 def run(self,
1463 def run(self,
1464 path=None,
1464 path=None,
1465 startDate=None,
1465 startDate=None,
1466 endDate=None,
1466 endDate=None,
1467 startTime=datetime.time(0, 0, 0),
1467 startTime=datetime.time(0, 0, 0),
1468 endTime=datetime.time(23, 59, 59),
1468 endTime=datetime.time(23, 59, 59),
1469 set=None,
1469 set=None,
1470 expLabel="",
1470 expLabel="",
1471 ext=None,
1471 ext=None,
1472 online=False,
1472 online=False,
1473 delay=60,
1473 delay=60,
1474 walk=True,
1474 walk=True,
1475 getblock=False,
1475 getblock=False,
1476 nTxs=1,
1476 nTxs=1,
1477 realtime=False,
1477 realtime=False,
1478 blocksize=None,
1478 blocksize=None,
1479 blocktime=None,
1479 blocktime=None,
1480 skip=None,
1480 skip=None,
1481 cursor=None,
1481 cursor=None,
1482 warnings=True,
1482 warnings=True,
1483 server=None,
1483 server=None,
1484 verbose=True,
1484 verbose=True,
1485 format=None,
1485 format=None,
1486 oneDDict=None,
1486 oneDDict=None,
1487 twoDDict=None,
1487 twoDDict=None,
1488 ind2DList=None, **kwargs):
1488 ind2DList=None, **kwargs):
1489
1489
1490 if not(self.isConfig):
1490 if not(self.isConfig):
1491 self.setup(path=path,
1491 self.setup(path=path,
1492 startDate=startDate,
1492 startDate=startDate,
1493 endDate=endDate,
1493 endDate=endDate,
1494 startTime=startTime,
1494 startTime=startTime,
1495 endTime=endTime,
1495 endTime=endTime,
1496 set=set,
1496 set=set,
1497 expLabel=expLabel,
1497 expLabel=expLabel,
1498 ext=ext,
1498 ext=ext,
1499 online=online,
1499 online=online,
1500 delay=delay,
1500 delay=delay,
1501 walk=walk,
1501 walk=walk,
1502 getblock=getblock,
1502 getblock=getblock,
1503 nTxs=nTxs,
1503 nTxs=nTxs,
1504 realtime=realtime,
1504 realtime=realtime,
1505 blocksize=blocksize,
1505 blocksize=blocksize,
1506 blocktime=blocktime,
1506 blocktime=blocktime,
1507 skip=skip,
1507 skip=skip,
1508 cursor=cursor,
1508 cursor=cursor,
1509 warnings=warnings,
1509 warnings=warnings,
1510 server=server,
1510 server=server,
1511 verbose=verbose,
1511 verbose=verbose,
1512 format=format,
1512 format=format,
1513 oneDDict=oneDDict,
1513 oneDDict=oneDDict,
1514 twoDDict=twoDDict,
1514 twoDDict=twoDDict,
1515 ind2DList=ind2DList)
1515 ind2DList=ind2DList)
1516 self.isConfig = True
1516 self.isConfig = True
1517 if server is None:
1517 if server is None:
1518 self.getData()
1518 self.getData()
1519 else:
1519 else:
1520 self.getFromServer()
1520 self.getFromServer()
1521
1521
1522
1522
1523 class JRODataWriter(JRODataIO):
1523 class JRODataWriter(JRODataIO):
1524
1524
1525 """
1525 """
1526 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1526 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1527 de los datos siempre se realiza por bloques.
1527 de los datos siempre se realiza por bloques.
1528 """
1528 """
1529
1529
1530 blockIndex = 0
1530 blockIndex = 0
1531
1531
1532 path = None
1532 path = None
1533
1533
1534 setFile = None
1534 setFile = None
1535
1535
1536 profilesPerBlock = None
1536 profilesPerBlock = None
1537
1537
1538 blocksPerFile = None
1538 blocksPerFile = None
1539
1539
1540 nWriteBlocks = 0
1540 nWriteBlocks = 0
1541
1541
1542 fileDate = None
1542 fileDate = None
1543
1543
1544 def __init__(self, dataOut=None):
1544 def __init__(self, dataOut=None):
1545 raise NotImplementedError
1545 raise NotImplementedError
1546
1546
1547 def hasAllDataInBuffer(self):
1547 def hasAllDataInBuffer(self):
1548 raise NotImplementedError
1548 raise NotImplementedError
1549
1549
1550 def setBlockDimension(self):
1550 def setBlockDimension(self):
1551 raise NotImplementedError
1551 raise NotImplementedError
1552
1552
1553 def writeBlock(self):
1553 def writeBlock(self):
1554 raise NotImplementedError
1554 raise NotImplementedError
1555
1555
1556 def putData(self):
1556 def putData(self):
1557 raise NotImplementedError
1557 raise NotImplementedError
1558
1558
1559 def getProcessFlags(self):
1559 def getProcessFlags(self):
1560
1560
1561 processFlags = 0
1561 processFlags = 0
1562
1562
1563 dtype_index = get_dtype_index(self.dtype)
1563 dtype_index = get_dtype_index(self.dtype)
1564 procflag_dtype = get_procflag_dtype(dtype_index)
1564 procflag_dtype = get_procflag_dtype(dtype_index)
1565
1565
1566 processFlags += procflag_dtype
1566 processFlags += procflag_dtype
1567
1567
1568 if self.dataOut.flagDecodeData:
1568 if self.dataOut.flagDecodeData:
1569 processFlags += PROCFLAG.DECODE_DATA
1569 processFlags += PROCFLAG.DECODE_DATA
1570
1570
1571 if self.dataOut.flagDeflipData:
1571 if self.dataOut.flagDeflipData:
1572 processFlags += PROCFLAG.DEFLIP_DATA
1572 processFlags += PROCFLAG.DEFLIP_DATA
1573
1573
1574 if self.dataOut.code is not None:
1574 if self.dataOut.code is not None:
1575 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1575 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1576
1576
1577 if self.dataOut.nCohInt > 1:
1577 if self.dataOut.nCohInt > 1:
1578 processFlags += PROCFLAG.COHERENT_INTEGRATION
1578 processFlags += PROCFLAG.COHERENT_INTEGRATION
1579
1579
1580 if self.dataOut.type == "Spectra":
1580 if self.dataOut.type == "Spectra":
1581 if self.dataOut.nIncohInt > 1:
1581 if self.dataOut.nIncohInt > 1:
1582 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1582 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1583
1583
1584 if self.dataOut.data_dc is not None:
1584 if self.dataOut.data_dc is not None:
1585 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1585 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1586
1586
1587 if self.dataOut.flagShiftFFT:
1587 if self.dataOut.flagShiftFFT:
1588 processFlags += PROCFLAG.SHIFT_FFT_DATA
1588 processFlags += PROCFLAG.SHIFT_FFT_DATA
1589
1589
1590 return processFlags
1590 return processFlags
1591
1591
1592 def setBasicHeader(self):
1592 def setBasicHeader(self):
1593
1593
1594 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1594 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1595 self.basicHeaderObj.version = self.versionFile
1595 self.basicHeaderObj.version = self.versionFile
1596 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1596 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1597
1597
1598 utc = numpy.floor(self.dataOut.utctime)
1598 utc = numpy.floor(self.dataOut.utctime)
1599 milisecond = (self.dataOut.utctime - utc) * 1000.0
1599 milisecond = (self.dataOut.utctime - utc) * 1000.0
1600
1600
1601 self.basicHeaderObj.utc = utc
1601 self.basicHeaderObj.utc = utc
1602 self.basicHeaderObj.miliSecond = milisecond
1602 self.basicHeaderObj.miliSecond = milisecond
1603 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1603 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1604 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1604 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1605 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1605 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1606
1606
1607 def setFirstHeader(self):
1607 def setFirstHeader(self):
1608 """
1608 """
1609 Obtiene una copia del First Header
1609 Obtiene una copia del First Header
1610
1610
1611 Affected:
1611 Affected:
1612
1612
1613 self.basicHeaderObj
1613 self.basicHeaderObj
1614 self.systemHeaderObj
1614 self.systemHeaderObj
1615 self.radarControllerHeaderObj
1615 self.radarControllerHeaderObj
1616 self.processingHeaderObj self.
1616 self.processingHeaderObj self.
1617
1617
1618 Return:
1618 Return:
1619 None
1619 None
1620 """
1620 """
1621
1621
1622 raise NotImplementedError
1622 raise NotImplementedError
1623
1623
1624 def __writeFirstHeader(self):
1624 def __writeFirstHeader(self):
1625 """
1625 """
1626 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1626 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1627
1627
1628 Affected:
1628 Affected:
1629 __dataType
1629 __dataType
1630
1630
1631 Return:
1631 Return:
1632 None
1632 None
1633 """
1633 """
1634
1634
1635 # CALCULAR PARAMETROS
1635 # CALCULAR PARAMETROS
1636
1636
1637 sizeLongHeader = self.systemHeaderObj.size + \
1637 sizeLongHeader = self.systemHeaderObj.size + \
1638 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1638 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1639 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1639 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1640
1640
1641 self.basicHeaderObj.write(self.fp)
1641 self.basicHeaderObj.write(self.fp)
1642 self.systemHeaderObj.write(self.fp)
1642 self.systemHeaderObj.write(self.fp)
1643 self.radarControllerHeaderObj.write(self.fp)
1643 self.radarControllerHeaderObj.write(self.fp)
1644 self.processingHeaderObj.write(self.fp)
1644 self.processingHeaderObj.write(self.fp)
1645
1645
1646 def __setNewBlock(self):
1646 def __setNewBlock(self):
1647 """
1647 """
1648 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1648 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1649
1649
1650 Return:
1650 Return:
1651 0 : si no pudo escribir nada
1651 0 : si no pudo escribir nada
1652 1 : Si escribio el Basic el First Header
1652 1 : Si escribio el Basic el First Header
1653 """
1653 """
1654 if self.fp == None:
1654 if self.fp == None:
1655 self.setNextFile()
1655 self.setNextFile()
1656
1656
1657 if self.flagIsNewFile:
1657 if self.flagIsNewFile:
1658 return 1
1658 return 1
1659
1659
1660 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1660 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1661 self.basicHeaderObj.write(self.fp)
1661 self.basicHeaderObj.write(self.fp)
1662 return 1
1662 return 1
1663
1663
1664 if not(self.setNextFile()):
1664 if not(self.setNextFile()):
1665 return 0
1665 return 0
1666
1666
1667 return 1
1667 return 1
1668
1668
1669 def writeNextBlock(self):
1669 def writeNextBlock(self):
1670 """
1670 """
1671 Selecciona el bloque siguiente de datos y los escribe en un file
1671 Selecciona el bloque siguiente de datos y los escribe en un file
1672
1672
1673 Return:
1673 Return:
1674 0 : Si no hizo pudo escribir el bloque de datos
1674 0 : Si no hizo pudo escribir el bloque de datos
1675 1 : Si no pudo escribir el bloque de datos
1675 1 : Si no pudo escribir el bloque de datos
1676 """
1676 """
1677 if not(self.__setNewBlock()):
1677 if not(self.__setNewBlock()):
1678 return 0
1678 return 0
1679
1679
1680 self.writeBlock()
1680 self.writeBlock()
1681
1681
1682 print("[Writing] Block No. %d/%d" % (self.blockIndex,
1682 print("[Writing] Block No. %d/%d" % (self.blockIndex,
1683 self.processingHeaderObj.dataBlocksPerFile))
1683 self.processingHeaderObj.dataBlocksPerFile))
1684
1684
1685 return 1
1685 return 1
1686
1686
1687 def setNextFile(self):
1687 def setNextFile(self):
1688 """
1688 """
1689 Determina el siguiente file que sera escrito
1689 Determina el siguiente file que sera escrito
1690
1690
1691 Affected:
1691 Affected:
1692 self.filename
1692 self.filename
1693 self.subfolder
1693 self.subfolder
1694 self.fp
1694 self.fp
1695 self.setFile
1695 self.setFile
1696 self.flagIsNewFile
1696 self.flagIsNewFile
1697
1697
1698 Return:
1698 Return:
1699 0 : Si el archivo no puede ser escrito
1699 0 : Si el archivo no puede ser escrito
1700 1 : Si el archivo esta listo para ser escrito
1700 1 : Si el archivo esta listo para ser escrito
1701 """
1701 """
1702 ext = self.ext
1702 ext = self.ext
1703 path = self.path
1703 path = self.path
1704
1704
1705 if self.fp != None:
1705 if self.fp != None:
1706 self.fp.close()
1706 self.fp.close()
1707
1707
1708 timeTuple = time.localtime(self.dataOut.utctime)
1708 timeTuple = time.localtime(self.dataOut.utctime)
1709 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1709 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1710
1710
1711 fullpath = os.path.join(path, subfolder)
1711 fullpath = os.path.join(path, subfolder)
1712 setFile = self.setFile
1712 setFile = self.setFile
1713
1713
1714 if not(os.path.exists(fullpath)):
1714 if not(os.path.exists(fullpath)):
1715 os.mkdir(fullpath)
1715 os.mkdir(fullpath)
1716 setFile = -1 # inicializo mi contador de seteo
1716 setFile = -1 # inicializo mi contador de seteo
1717 else:
1717 else:
1718 filesList = os.listdir(fullpath)
1718 filesList = os.listdir(fullpath)
1719 if len(filesList) > 0:
1719 if len(filesList) > 0:
1720 filesList = sorted(filesList, key=str.lower)
1720 filesList = sorted(filesList, key=str.lower)
1721 filen = filesList[-1]
1721 filen = filesList[-1]
1722 # el filename debera tener el siguiente formato
1722 # el filename debera tener el siguiente formato
1723 # 0 1234 567 89A BCDE (hex)
1723 # 0 1234 567 89A BCDE (hex)
1724 # x YYYY DDD SSS .ext
1724 # x YYYY DDD SSS .ext
1725 if isNumber(filen[8:11]):
1725 if isNumber(filen[8:11]):
1726 # inicializo mi contador de seteo al seteo del ultimo file
1726 # inicializo mi contador de seteo al seteo del ultimo file
1727 setFile = int(filen[8:11])
1727 setFile = int(filen[8:11])
1728 else:
1728 else:
1729 setFile = -1
1729 setFile = -1
1730 else:
1730 else:
1731 setFile = -1 # inicializo mi contador de seteo
1731 setFile = -1 # inicializo mi contador de seteo
1732
1732
1733 setFile += 1
1733 setFile += 1
1734
1734
1735 # If this is a new day it resets some values
1735 # If this is a new day it resets some values
1736 if self.dataOut.datatime.date() > self.fileDate:
1736 if self.dataOut.datatime.date() > self.fileDate:
1737 setFile = 0
1737 setFile = 0
1738 self.nTotalBlocks = 0
1738 self.nTotalBlocks = 0
1739
1739
1740 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1740 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1741 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1741 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1742
1742
1743 filename = os.path.join(path, subfolder, filen)
1743 filename = os.path.join(path, subfolder, filen)
1744
1744
1745 fp = open(filename, 'wb')
1745 fp = open(filename, 'wb')
1746
1746
1747 self.blockIndex = 0
1747 self.blockIndex = 0
1748
1748
1749 # guardando atributos
1749 # guardando atributos
1750 self.filename = filename
1750 self.filename = filename
1751 self.subfolder = subfolder
1751 self.subfolder = subfolder
1752 self.fp = fp
1752 self.fp = fp
1753 self.setFile = setFile
1753 self.setFile = setFile
1754 self.flagIsNewFile = 1
1754 self.flagIsNewFile = 1
1755 self.fileDate = self.dataOut.datatime.date()
1755 self.fileDate = self.dataOut.datatime.date()
1756
1756
1757 self.setFirstHeader()
1757 self.setFirstHeader()
1758
1758
1759 print('[Writing] Opening file: %s' % self.filename)
1759 print('[Writing] Opening file: %s' % self.filename)
1760
1760
1761 self.__writeFirstHeader()
1761 self.__writeFirstHeader()
1762
1762
1763 return 1
1763 return 1
1764
1764
1765 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1765 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1766 """
1766 """
1767 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1767 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1768
1768
1769 Inputs:
1769 Inputs:
1770 path : directory where data will be saved
1770 path : directory where data will be saved
1771 profilesPerBlock : number of profiles per block
1771 profilesPerBlock : number of profiles per block
1772 set : initial file set
1772 set : initial file set
1773 datatype : An integer number that defines data type:
1773 datatype : An integer number that defines data type:
1774 0 : int8 (1 byte)
1774 0 : int8 (1 byte)
1775 1 : int16 (2 bytes)
1775 1 : int16 (2 bytes)
1776 2 : int32 (4 bytes)
1776 2 : int32 (4 bytes)
1777 3 : int64 (8 bytes)
1777 3 : int64 (8 bytes)
1778 4 : float32 (4 bytes)
1778 4 : float32 (4 bytes)
1779 5 : double64 (8 bytes)
1779 5 : double64 (8 bytes)
1780
1780
1781 Return:
1781 Return:
1782 0 : Si no realizo un buen seteo
1782 0 : Si no realizo un buen seteo
1783 1 : Si realizo un buen seteo
1783 1 : Si realizo un buen seteo
1784 """
1784 """
1785
1785
1786 if ext == None:
1786 if ext == None:
1787 ext = self.ext
1787 ext = self.ext
1788
1788
1789 self.ext = ext.lower()
1789 self.ext = ext.lower()
1790
1790
1791 self.path = path
1791 self.path = path
1792
1792
1793 if set is None:
1793 if set is None:
1794 self.setFile = -1
1794 self.setFile = -1
1795 else:
1795 else:
1796 self.setFile = set - 1
1796 self.setFile = set - 1
1797
1797
1798 self.blocksPerFile = blocksPerFile
1798 self.blocksPerFile = blocksPerFile
1799
1799
1800 self.profilesPerBlock = profilesPerBlock
1800 self.profilesPerBlock = profilesPerBlock
1801
1801
1802 self.dataOut = dataOut
1802 self.dataOut = dataOut
1803 self.fileDate = self.dataOut.datatime.date()
1803 self.fileDate = self.dataOut.datatime.date()
1804 # By default
1804 # By default
1805 self.dtype = self.dataOut.dtype
1805 self.dtype = self.dataOut.dtype
1806
1806
1807 if datatype is not None:
1807 if datatype is not None:
1808 self.dtype = get_numpy_dtype(datatype)
1808 self.dtype = get_numpy_dtype(datatype)
1809
1809
1810 if not(self.setNextFile()):
1810 if not(self.setNextFile()):
1811 print("[Writing] There isn't a next file")
1811 print("[Writing] There isn't a next file")
1812 return 0
1812 return 0
1813
1813
1814 self.setBlockDimension()
1814 self.setBlockDimension()
1815
1815
1816 return 1
1816 return 1
1817
1817
1818 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1818 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1819
1819
1820 if not(self.isConfig):
1820 if not(self.isConfig):
1821
1821
1822 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1822 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1823 set=set, ext=ext, datatype=datatype, **kwargs)
1823 set=set, ext=ext, datatype=datatype, **kwargs)
1824 self.isConfig = True
1824 self.isConfig = True
1825
1825
1826 self.putData() No newline at end of file
1826 self.dataOut = dataOut
1827 self.putData()
1828 return self.dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1095 +1,1103
1 import numpy
1 import numpy
2 import time
2 import time
3 import os
3 import os
4 import h5py
4 import h5py
5 import re
5 import re
6 import datetime
6 import datetime
7
7
8 from schainpy.model.data.jrodata import *
8 from schainpy.model.data.jrodata import *
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 # from .jroIO_base import *
10 # from .jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 import schainpy
12 import schainpy
13 from schainpy.utils import log
13
14
14
15 @MPDecorator
15 class ParamReader(ProcessingUnit):
16 class ParamReader(JRODataReader,ProcessingUnit):
16 '''
17 '''
17 Reads HDF5 format files
18 Reads HDF5 format files
18
19
19 path
20 path
20
21
21 startDate
22 startDate
22
23
23 endDate
24 endDate
24
25
25 startTime
26 startTime
26
27
27 endTime
28 endTime
28 '''
29 '''
29
30
30 ext = ".hdf5"
31 ext = ".hdf5"
31
32
32 optchar = "D"
33 optchar = "D"
33
34
34 timezone = None
35 timezone = None
35
36
36 startTime = None
37 startTime = None
37
38
38 endTime = None
39 endTime = None
39
40
40 fileIndex = None
41 fileIndex = None
41
42
42 utcList = None #To select data in the utctime list
43 utcList = None #To select data in the utctime list
43
44
44 blockList = None #List to blocks to be read from the file
45 blockList = None #List to blocks to be read from the file
45
46
46 blocksPerFile = None #Number of blocks to be read
47 blocksPerFile = None #Number of blocks to be read
47
48
48 blockIndex = None
49 blockIndex = None
49
50
50 path = None
51 path = None
51
52
52 #List of Files
53 #List of Files
53
54
54 filenameList = None
55 filenameList = None
55
56
56 datetimeList = None
57 datetimeList = None
57
58
58 #Hdf5 File
59 #Hdf5 File
59
60
60 listMetaname = None
61 listMetaname = None
61
62
62 listMeta = None
63 listMeta = None
63
64
64 listDataname = None
65 listDataname = None
65
66
66 listData = None
67 listData = None
67
68
68 listShapes = None
69 listShapes = None
69
70
70 fp = None
71 fp = None
71
72
72 #dataOut reconstruction
73 #dataOut reconstruction
73
74
74 dataOut = None
75 dataOut = None
75
76
76
77
77 def __init__(self, **kwargs):
78 def __init__(self):#, **kwargs):
78 ProcessingUnit.__init__(self, **kwargs)
79 ProcessingUnit.__init__(self) #, **kwargs)
79 self.dataOut = Parameters()
80 self.dataOut = Parameters()
80 return
81 return
81
82
82 def setup(self, **kwargs):
83 def setup(self, **kwargs):
83
84
84 path = kwargs['path']
85 path = kwargs['path']
85 startDate = kwargs['startDate']
86 startDate = kwargs['startDate']
86 endDate = kwargs['endDate']
87 endDate = kwargs['endDate']
87 startTime = kwargs['startTime']
88 startTime = kwargs['startTime']
88 endTime = kwargs['endTime']
89 endTime = kwargs['endTime']
89 walk = kwargs['walk']
90 walk = kwargs['walk']
90 if 'ext' in kwargs:
91 if 'ext' in kwargs:
91 ext = kwargs['ext']
92 ext = kwargs['ext']
92 else:
93 else:
93 ext = '.hdf5'
94 ext = '.hdf5'
94 if 'timezone' in kwargs:
95 if 'timezone' in kwargs:
95 self.timezone = kwargs['timezone']
96 self.timezone = kwargs['timezone']
96 else:
97 else:
97 self.timezone = 'lt'
98 self.timezone = 'lt'
98
99
99 print("[Reading] Searching files in offline mode ...")
100 print("[Reading] Searching files in offline mode ...")
100 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 startTime=startTime, endTime=endTime,
102 startTime=startTime, endTime=endTime,
102 ext=ext, walk=walk)
103 ext=ext, walk=walk)
103
104
104 if not(filenameList):
105 if not(filenameList):
105 print("There is no files into the folder: %s"%(path))
106 print("There is no files into the folder: %s"%(path))
106 sys.exit(-1)
107 sys.exit(-1)
107
108
108 self.fileIndex = -1
109 self.fileIndex = -1
109 self.startTime = startTime
110 self.startTime = startTime
110 self.endTime = endTime
111 self.endTime = endTime
111
112
112 self.__readMetadata()
113 self.__readMetadata()
113
114
114 self.__setNextFileOffline()
115 self.__setNextFileOffline()
115
116
116 return
117 return
117
118
118 def searchFilesOffLine(self,
119 def searchFilesOffLine(self,
119 path,
120 path,
120 startDate=None,
121 startDate=None,
121 endDate=None,
122 endDate=None,
122 startTime=datetime.time(0,0,0),
123 startTime=datetime.time(0,0,0),
123 endTime=datetime.time(23,59,59),
124 endTime=datetime.time(23,59,59),
124 ext='.hdf5',
125 ext='.hdf5',
125 walk=True):
126 walk=True):
126
127
127 expLabel = ''
128 expLabel = ''
128 self.filenameList = []
129 self.filenameList = []
129 self.datetimeList = []
130 self.datetimeList = []
130
131
131 pathList = []
132 pathList = []
132
133
133 JRODataObj = JRODataReader()
134 JRODataObj = JRODataReader()
134 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
135 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
135
136
136 if dateList == []:
137 if dateList == []:
137 print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
138 print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
138 datetime.datetime.combine(startDate,startTime).ctime(),
139 datetime.datetime.combine(startDate,startTime).ctime(),
139 datetime.datetime.combine(endDate,endTime).ctime()))
140 datetime.datetime.combine(endDate,endTime).ctime()))
140
141
141 return None, None
142 return None, None
142
143
143 if len(dateList) > 1:
144 if len(dateList) > 1:
144 print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
145 print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
145 else:
146 else:
146 print("[Reading] data was found for the date %s" %(dateList[0]))
147 print("[Reading] data was found for the date %s" %(dateList[0]))
147
148
148 filenameList = []
149 filenameList = []
149 datetimeList = []
150 datetimeList = []
150
151
151 #----------------------------------------------------------------------------------
152 #----------------------------------------------------------------------------------
152
153
153 for thisPath in pathList:
154 for thisPath in pathList:
154 # thisPath = pathList[pathDict[file]]
155 # thisPath = pathList[pathDict[file]]
155
156
156 fileList = glob.glob1(thisPath, "*%s" %ext)
157 fileList = glob.glob1(thisPath, "*%s" %ext)
157 fileList.sort()
158 fileList.sort()
158
159
159 for file in fileList:
160 for file in fileList:
160
161
161 filename = os.path.join(thisPath,file)
162 filename = os.path.join(thisPath,file)
162
163
163 if not isFileInDateRange(filename, startDate, endDate):
164 if not isFileInDateRange(filename, startDate, endDate):
164 continue
165 continue
165
166
166 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
167 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
167
168
168 if not(thisDatetime):
169 if not(thisDatetime):
169 continue
170 continue
170
171
171 filenameList.append(filename)
172 filenameList.append(filename)
172 datetimeList.append(thisDatetime)
173 datetimeList.append(thisDatetime)
173
174
174 if not(filenameList):
175 if not(filenameList):
175 print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
176 print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
176 return None, None
177 return None, None
177
178
178 print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
179 print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
179 print()
180 print()
180
181
181 # for i in range(len(filenameList)):
182 # for i in range(len(filenameList)):
182 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
183 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
183
184
184 self.filenameList = filenameList
185 self.filenameList = filenameList
185 self.datetimeList = datetimeList
186 self.datetimeList = datetimeList
186
187
187 return pathList, filenameList
188 return pathList, filenameList
188
189
189 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
190 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
190
191
191 """
192 """
192 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
193 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
193
194
194 Inputs:
195 Inputs:
195 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
196 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
196
197
197 startDate : fecha inicial del rango seleccionado en formato datetime.date
198 startDate : fecha inicial del rango seleccionado en formato datetime.date
198
199
199 endDate : fecha final del rango seleccionado en formato datetime.date
200 endDate : fecha final del rango seleccionado en formato datetime.date
200
201
201 startTime : tiempo inicial del rango seleccionado en formato datetime.time
202 startTime : tiempo inicial del rango seleccionado en formato datetime.time
202
203
203 endTime : tiempo final del rango seleccionado en formato datetime.time
204 endTime : tiempo final del rango seleccionado en formato datetime.time
204
205
205 Return:
206 Return:
206 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
207 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
207 fecha especificado, de lo contrario retorna False.
208 fecha especificado, de lo contrario retorna False.
208
209
209 Excepciones:
210 Excepciones:
210 Si el archivo no existe o no puede ser abierto
211 Si el archivo no existe o no puede ser abierto
211 Si la cabecera no puede ser leida.
212 Si la cabecera no puede ser leida.
212
213
213 """
214 """
214
215
215 try:
216 try:
216 fp = h5py.File(filename,'r')
217 fp = h5py.File(filename,'r')
217 grp1 = fp['Data']
218 grp1 = fp['Data']
218
219
219 except IOError:
220 except IOError:
220 traceback.print_exc()
221 traceback.print_exc()
221 raise IOError("The file %s can't be opened" %(filename))
222 raise IOError("The file %s can't be opened" %(filename))
222 #chino rata
223 #chino rata
223 #In case has utctime attribute
224 #In case has utctime attribute
224 grp2 = grp1['utctime']
225 grp2 = grp1['utctime']
225 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
226 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
226 thisUtcTime = grp2.value[0]
227 thisUtcTime = grp2.value[0]
227
228
228 fp.close()
229 fp.close()
229
230
230 if self.timezone == 'lt':
231 if self.timezone == 'lt':
231 thisUtcTime -= 5*3600
232 thisUtcTime -= 5*3600
232
233
233 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
234 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
234 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
235 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
235 thisDate = thisDatetime.date()
236 thisDate = thisDatetime.date()
236 thisTime = thisDatetime.time()
237 thisTime = thisDatetime.time()
237
238
238 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
240 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
240
241
241 #General case
242 #General case
242 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
243 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
243 #-----------o----------------------------o-----------
244 #-----------o----------------------------o-----------
244 # startTime endTime
245 # startTime endTime
245
246
246 if endTime >= startTime:
247 if endTime >= startTime:
247 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
248 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
248 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
249 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
249 return thisDatetime
250 return thisDatetime
250 return None
251 return None
251
252
252 #If endTime < startTime then endTime belongs to the next day
253 #If endTime < startTime then endTime belongs to the next day
253 #<<<<<<<<<<<o o>>>>>>>>>>>
254 #<<<<<<<<<<<o o>>>>>>>>>>>
254 #-----------o----------------------------o-----------
255 #-----------o----------------------------o-----------
255 # endTime startTime
256 # endTime startTime
256
257
257 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
258 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
258 return None
259 return None
259
260
260 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
261 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
261 return None
262 return None
262
263
263 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
264 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
264 return None
265 return None
265
266
266 return thisDatetime
267 return thisDatetime
267
268
268 def __setNextFileOffline(self):
269 def __setNextFileOffline(self):
269
270
270 self.fileIndex += 1
271 self.fileIndex += 1
271 idFile = self.fileIndex
272 idFile = self.fileIndex
272
273
273 if not(idFile < len(self.filenameList)):
274 if not(idFile < len(self.filenameList)):
274 print("No more Files")
275 print("No more Files")
275 return 0
276 return 0
276
277
277 filename = self.filenameList[idFile]
278 filename = self.filenameList[idFile]
278
279
279 filePointer = h5py.File(filename,'r')
280 filePointer = h5py.File(filename,'r')
280
281
281 self.filename = filename
282 self.filename = filename
282
283
283 self.fp = filePointer
284 self.fp = filePointer
284
285
285 print("Setting the file: %s"%self.filename)
286 print("Setting the file: %s"%self.filename)
286
287
287 # self.__readMetadata()
288 # self.__readMetadata()
288 self.__setBlockList()
289 self.__setBlockList()
289 self.__readData()
290 self.__readData()
290 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
291 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
291 # self.nRecords = self.fp['Data'].attrs['nRecords']
292 # self.nRecords = self.fp['Data'].attrs['nRecords']
292 self.blockIndex = 0
293 self.blockIndex = 0
293 return 1
294 return 1
294
295
295 def __setBlockList(self):
296 def __setBlockList(self):
296 '''
297 '''
297 Selects the data within the times defined
298 Selects the data within the times defined
298
299
299 self.fp
300 self.fp
300 self.startTime
301 self.startTime
301 self.endTime
302 self.endTime
302
303
303 self.blockList
304 self.blockList
304 self.blocksPerFile
305 self.blocksPerFile
305
306
306 '''
307 '''
307 fp = self.fp
308 fp = self.fp
308 startTime = self.startTime
309 startTime = self.startTime
309 endTime = self.endTime
310 endTime = self.endTime
310
311
311 grp = fp['Data']
312 grp = fp['Data']
312 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
313 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
313
314
314 #ERROOOOR
315 #ERROOOOR
315 if self.timezone == 'lt':
316 if self.timezone == 'lt':
316 thisUtcTime -= 5*3600
317 thisUtcTime -= 5*3600
317
318
318 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
319 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
319
320
320 thisDate = thisDatetime.date()
321 thisDate = thisDatetime.date()
321 thisTime = thisDatetime.time()
322 thisTime = thisDatetime.time()
322
323
323 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
325 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
325
326
326 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
327 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
327
328
328 self.blockList = ind
329 self.blockList = ind
329 self.blocksPerFile = len(ind)
330 self.blocksPerFile = len(ind)
330
331
331 return
332 return
332
333
333 def __readMetadata(self):
334 def __readMetadata(self):
334 '''
335 '''
335 Reads Metadata
336 Reads Metadata
336
337
337 self.pathMeta
338 self.pathMeta
338
339
339 self.listShapes
340 self.listShapes
340 self.listMetaname
341 self.listMetaname
341 self.listMeta
342 self.listMeta
342
343
343 '''
344 '''
344
345
345 # grp = self.fp['Data']
346 # grp = self.fp['Data']
346 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
347 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
347 #
348 #
348 # if pathMeta == self.pathMeta:
349 # if pathMeta == self.pathMeta:
349 # return
350 # return
350 # else:
351 # else:
351 # self.pathMeta = pathMeta
352 # self.pathMeta = pathMeta
352 #
353 #
353 # filePointer = h5py.File(self.pathMeta,'r')
354 # filePointer = h5py.File(self.pathMeta,'r')
354 # groupPointer = filePointer['Metadata']
355 # groupPointer = filePointer['Metadata']
355
356
356 filename = self.filenameList[0]
357 filename = self.filenameList[0]
357
358
358 fp = h5py.File(filename,'r')
359 fp = h5py.File(filename,'r')
359
360
360 gp = fp['Metadata']
361 gp = fp['Metadata']
361
362
362 listMetaname = []
363 listMetaname = []
363 listMetadata = []
364 listMetadata = []
364 for item in list(gp.items()):
365 for item in list(gp.items()):
365 name = item[0]
366 name = item[0]
366
367
367 if name=='array dimensions':
368 if name=='array dimensions':
368 table = gp[name][:]
369 table = gp[name][:]
369 listShapes = {}
370 listShapes = {}
370 for shapes in table:
371 for shapes in table:
371 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
372 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
372 else:
373 else:
373 data = gp[name].value
374 data = gp[name].value
374 listMetaname.append(name)
375 listMetaname.append(name)
375 listMetadata.append(data)
376 listMetadata.append(data)
376
377
377 # if name=='type':
378 # if name=='type':
378 # self.__initDataOut(data)
379 # self.__initDataOut(data)
379
380
380 self.listShapes = listShapes
381 self.listShapes = listShapes
381 self.listMetaname = listMetaname
382 self.listMetaname = listMetaname
382 self.listMeta = listMetadata
383 self.listMeta = listMetadata
383
384
384 fp.close()
385 fp.close()
385 return
386 return
386
387
387 def __readData(self):
388 def __readData(self):
388 grp = self.fp['Data']
389 grp = self.fp['Data']
389 listdataname = []
390 listdataname = []
390 listdata = []
391 listdata = []
391
392
392 for item in list(grp.items()):
393 for item in list(grp.items()):
393 name = item[0]
394 name = item[0]
394 listdataname.append(name)
395 listdataname.append(name)
395
396
396 array = self.__setDataArray(grp[name],self.listShapes[name])
397 array = self.__setDataArray(grp[name],self.listShapes[name])
397 listdata.append(array)
398 listdata.append(array)
398
399
399 self.listDataname = listdataname
400 self.listDataname = listdataname
400 self.listData = listdata
401 self.listData = listdata
401 return
402 return
402
403
403 def __setDataArray(self, dataset, shapes):
404 def __setDataArray(self, dataset, shapes):
404
405
405 nDims = shapes[0]
406 nDims = shapes[0]
406
407
407 nDim2 = shapes[1] #Dimension 0
408 nDim2 = shapes[1] #Dimension 0
408
409
409 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
410 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
410
411
411 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
412 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
412
413
413 mode = shapes[4] #Mode of storing
414 mode = shapes[4] #Mode of storing
414
415
415 blockList = self.blockList
416 blockList = self.blockList
416
417
417 blocksPerFile = self.blocksPerFile
418 blocksPerFile = self.blocksPerFile
418
419
419 #Depending on what mode the data was stored
420 #Depending on what mode the data was stored
420 if mode == 0: #Divided in channels
421 if mode == 0: #Divided in channels
421 arrayData = dataset.value.astype(numpy.float)[0][blockList]
422 arrayData = dataset.value.astype(numpy.float)[0][blockList]
422 if mode == 1: #Divided in parameter
423 if mode == 1: #Divided in parameter
423 strds = 'table'
424 strds = 'table'
424 nDatas = nDim1
425 nDatas = nDim1
425 newShapes = (blocksPerFile,nDim2,nDim0)
426 newShapes = (blocksPerFile,nDim2,nDim0)
426 elif mode==2: #Concatenated in a table
427 elif mode==2: #Concatenated in a table
427 strds = 'table0'
428 strds = 'table0'
428 arrayData = dataset[strds].value
429 arrayData = dataset[strds].value
429 #Selecting part of the dataset
430 #Selecting part of the dataset
430 utctime = arrayData[:,0]
431 utctime = arrayData[:,0]
431 u, indices = numpy.unique(utctime, return_index=True)
432 u, indices = numpy.unique(utctime, return_index=True)
432
433
433 if blockList.size != indices.size:
434 if blockList.size != indices.size:
434 indMin = indices[blockList[0]]
435 indMin = indices[blockList[0]]
435 if blockList[1] + 1 >= indices.size:
436 if blockList[1] + 1 >= indices.size:
436 arrayData = arrayData[indMin:,:]
437 arrayData = arrayData[indMin:,:]
437 else:
438 else:
438 indMax = indices[blockList[1] + 1]
439 indMax = indices[blockList[1] + 1]
439 arrayData = arrayData[indMin:indMax,:]
440 arrayData = arrayData[indMin:indMax,:]
440 return arrayData
441 return arrayData
441
442
442 # One dimension
443 # One dimension
443 if nDims == 0:
444 if nDims == 0:
444 arrayData = dataset.value.astype(numpy.float)[0][blockList]
445 arrayData = dataset.value.astype(numpy.float)[0][blockList]
445
446
446 # Two dimensions
447 # Two dimensions
447 elif nDims == 2:
448 elif nDims == 2:
448 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
449 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
449 newShapes = (blocksPerFile,nDim0)
450 newShapes = (blocksPerFile,nDim0)
450 nDatas = nDim1
451 nDatas = nDim1
451
452
452 for i in range(nDatas):
453 for i in range(nDatas):
453 data = dataset[strds + str(i)].value
454 data = dataset[strds + str(i)].value
454 arrayData[:,i,:] = data[blockList,:]
455 arrayData[:,i,:] = data[blockList,:]
455
456
456 # Three dimensions
457 # Three dimensions
457 else:
458 else:
458 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
459 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
459 for i in range(nDatas):
460 for i in range(nDatas):
460
461
461 data = dataset[strds + str(i)].value
462 data = dataset[strds + str(i)].value
462
463
463 for b in range(blockList.size):
464 for b in range(blockList.size):
464 arrayData[b,:,i,:] = data[:,:,blockList[b]]
465 arrayData[b,:,i,:] = data[:,:,blockList[b]]
465
466
466 return arrayData
467 return arrayData
467
468
468 def __setDataOut(self):
469 def __setDataOut(self):
469 listMeta = self.listMeta
470 listMeta = self.listMeta
470 listMetaname = self.listMetaname
471 listMetaname = self.listMetaname
471 listDataname = self.listDataname
472 listDataname = self.listDataname
472 listData = self.listData
473 listData = self.listData
473 listShapes = self.listShapes
474 listShapes = self.listShapes
474
475
475 blockIndex = self.blockIndex
476 blockIndex = self.blockIndex
476 # blockList = self.blockList
477 # blockList = self.blockList
477
478
478 for i in range(len(listMeta)):
479 for i in range(len(listMeta)):
479 setattr(self.dataOut,listMetaname[i],listMeta[i])
480 setattr(self.dataOut,listMetaname[i],listMeta[i])
480
481
481 for j in range(len(listData)):
482 for j in range(len(listData)):
482 nShapes = listShapes[listDataname[j]][0]
483 nShapes = listShapes[listDataname[j]][0]
483 mode = listShapes[listDataname[j]][4]
484 mode = listShapes[listDataname[j]][4]
484 if nShapes == 1:
485 if nShapes == 1:
485 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
486 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
486 elif nShapes > 1:
487 elif nShapes > 1:
487 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
488 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
488 elif mode==0:
489 elif mode==0:
489 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
490 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
490 #Mode Meteors
491 #Mode Meteors
491 elif mode ==2:
492 elif mode ==2:
492 selectedData = self.__selectDataMode2(listData[j], blockIndex)
493 selectedData = self.__selectDataMode2(listData[j], blockIndex)
493 setattr(self.dataOut, listDataname[j], selectedData)
494 setattr(self.dataOut, listDataname[j], selectedData)
494 return
495 return
495
496
496 def __selectDataMode2(self, data, blockIndex):
497 def __selectDataMode2(self, data, blockIndex):
497 utctime = data[:,0]
498 utctime = data[:,0]
498 aux, indices = numpy.unique(utctime, return_inverse=True)
499 aux, indices = numpy.unique(utctime, return_inverse=True)
499 selInd = numpy.where(indices == blockIndex)[0]
500 selInd = numpy.where(indices == blockIndex)[0]
500 selData = data[selInd,:]
501 selData = data[selInd,:]
501
502
502 return selData
503 return selData
503
504
504 def getData(self):
505 def getData(self):
505
506
506 # if self.flagNoMoreFiles:
507 # if self.flagNoMoreFiles:
507 # self.dataOut.flagNoData = True
508 # self.dataOut.flagNoData = True
508 # print 'Process finished'
509 # print 'Process finished'
509 # return 0
510 # return 0
510 #
511 #
511 if self.blockIndex==self.blocksPerFile:
512 if self.blockIndex==self.blocksPerFile:
512 if not( self.__setNextFileOffline() ):
513 if not( self.__setNextFileOffline() ):
513 self.dataOut.flagNoData = True
514 self.dataOut.flagNoData = True
514 return 0
515 return 0
515
516
516 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
517 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
517 # self.dataOut.flagNoData = True
518 # self.dataOut.flagNoData = True
518 # return 0
519 # return 0
519 # self.__readData()
520 # self.__readData()
520 self.__setDataOut()
521 self.__setDataOut()
521 self.dataOut.flagNoData = False
522 self.dataOut.flagNoData = False
522
523
523 self.blockIndex += 1
524 self.blockIndex += 1
524
525
525 return
526 return
526
527
527 def run(self, **kwargs):
528 def run(self, **kwargs):
528
529
529 if not(self.isConfig):
530 if not(self.isConfig):
530 self.setup(**kwargs)
531 self.setup(**kwargs)
531 # self.setObjProperties()
532 # self.setObjProperties()
532 self.isConfig = True
533 self.isConfig = True
533
534
534 self.getData()
535 self.getData()
535
536
536 return
537 return
537
538 @MPDecorator
538 class ParamWriter(Operation):
539 class ParamWriter(Operation):
539 '''
540 '''
540 HDF5 Writer, stores parameters data in HDF5 format files
541 HDF5 Writer, stores parameters data in HDF5 format files
541
542
542 path: path where the files will be stored
543 path: path where the files will be stored
543
544
544 blocksPerFile: number of blocks that will be saved in per HDF5 format file
545 blocksPerFile: number of blocks that will be saved in per HDF5 format file
545
546
546 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
547 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
547
548
548 metadataList: list of attributes that will be stored as metadata
549 metadataList: list of attributes that will be stored as metadata
549
550
550 dataList: list of attributes that will be stores as data
551 dataList: list of attributes that will be stores as data
551
552
552 '''
553 '''
553
554
554
555
555 ext = ".hdf5"
556 ext = ".hdf5"
556
557
557 optchar = "D"
558 optchar = "D"
558
559
559 metaoptchar = "M"
560 metaoptchar = "M"
560
561
561 metaFile = None
562 metaFile = None
562
563
563 filename = None
564 filename = None
564
565
565 path = None
566 path = None
566
567
567 setFile = None
568 setFile = None
568
569
569 fp = None
570 fp = None
570
571
571 grp = None
572 grp = None
572
573
573 ds = None
574 ds = None
574
575
575 firsttime = True
576 firsttime = True
576
577
577 #Configurations
578 #Configurations
578
579
579 blocksPerFile = None
580 blocksPerFile = None
580
581
581 blockIndex = None
582 blockIndex = None
582
583
583 dataOut = None
584 dataOut = None
584
585
585 #Data Arrays
586 #Data Arrays
586
587
587 dataList = None
588 dataList = None
588
589
589 metadataList = None
590 metadataList = None
590
591
591 # arrayDim = None
592 # arrayDim = None
592
593
593 dsList = None #List of dictionaries with dataset properties
594 dsList = None #List of dictionaries with dataset properties
594
595
595 tableDim = None
596 tableDim = None
596
597
597 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
598 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
598
599
599 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
600 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
600
601
601 currentDay = None
602 currentDay = None
602
603
603 lastTime = None
604 lastTime = None
604
605
605 def __init__(self, **kwargs):
606 def __init__(self):#, **kwargs):
606 Operation.__init__(self, **kwargs)
607 Operation.__init__(self)#, **kwargs)
607 self.isConfig = False
608 #self.isConfig = False
608 return
609 return
609
610
610 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
611 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
611 self.path = path
612 self.path = path
612 self.blocksPerFile = blocksPerFile
613 self.blocksPerFile = blocksPerFile
613 self.metadataList = metadataList
614 self.metadataList = metadataList
614 self.dataList = dataList
615 self.dataList = dataList
615 self.dataOut = dataOut
616 self.dataOut = dataOut
616 self.mode = mode
617 self.mode = mode
617
618 if self.mode is not None:
618 if self.mode is not None:
619 self.mode = numpy.zeros(len(self.dataList)) + mode
619 self.mode = numpy.zeros(len(self.dataList)) + mode
620 else:
620 else:
621 #self.mode = numpy.ones(len(self.dataList),int)
621 self.mode = numpy.ones(len(self.dataList))
622 self.mode = numpy.ones(len(self.dataList))
623 log.error(self.mode)#yong
622
624
623 arrayDim = numpy.zeros((len(self.dataList),5))
625 arrayDim = numpy.zeros((len(self.dataList),5))
624
626
625 #Table dimensions
627 #Table dimensions
626 dtype0 = self.dtype
628 dtype0 = self.dtype
627 tableList = []
629 tableList = []
628
630
629 #Dictionary and list of tables
631 #Dictionary and list of tables
630 dsList = []
632 dsList = []
631
633
632 for i in range(len(self.dataList)):
634 for i in range(len(self.dataList)):
633 dsDict = {}
635 dsDict = {}
634 dataAux = getattr(self.dataOut, self.dataList[i])
636 dataAux = getattr(self.dataOut, self.dataList[i])
635 dsDict['variable'] = self.dataList[i]
637 dsDict['variable'] = self.dataList[i]
636 #--------------------- Conditionals ------------------------
638 #--------------------- Conditionals ------------------------
637 #There is no data
639 #There is no data
638 if dataAux is None:
640 if dataAux is None:
639 return 0
641 return 0
640
642
641 #Not array, just a number
643 #Not array, just a number
642 #Mode 0
644 #Mode 0
645 #log.error(mode)#yong
646 #log.error(len(mode))#yong
647 #log.error(type(mode))#yong
643 if type(dataAux)==float or type(dataAux)==int:
648 if type(dataAux)==float or type(dataAux)==int:
644 dsDict['mode'] = 0
649 dsDict['mode'] = 0
645 dsDict['nDim'] = 0
650 dsDict['nDim'] = 0
646 arrayDim[i,0] = 0
651 arrayDim[i,0] = 0
647 dsList.append(dsDict)
652 dsList.append(dsDict)
648
653
649 #Mode 2: meteors
654 #Mode 2: meteors
650 elif mode[i] == 2:
655 elif self.mode[i] == 2:
651 # dsDict['nDim'] = 0
656 # dsDict['nDim'] = 0
652 dsDict['dsName'] = 'table0'
657 dsDict['dsName'] = 'table0'
653 dsDict['mode'] = 2 # Mode meteors
658 dsDict['mode'] = 2 # Mode meteors
654 dsDict['shape'] = dataAux.shape[-1]
659 dsDict['shape'] = dataAux.shape[-1]
655 dsDict['nDim'] = 0
660 dsDict['nDim'] = 0
656 dsDict['dsNumber'] = 1
661 dsDict['dsNumber'] = 1
657
662
658 arrayDim[i,3] = dataAux.shape[-1]
663 arrayDim[i,3] = dataAux.shape[-1]
659 arrayDim[i,4] = mode[i] #Mode the data was stored
664 arrayDim[i,4] = self.mode[i] #Mode the data was stored
660
665
661 dsList.append(dsDict)
666 dsList.append(dsDict)
662
667
663 #Mode 1
668 #Mode 1
664 else:
669 else:
665 arrayDim0 = dataAux.shape #Data dimensions
670 arrayDim0 = dataAux.shape #Data dimensions
666 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
671 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
667 arrayDim[i,4] = mode[i] #Mode the data was stored
672 arrayDim[i,4] = self.mode[i] #Mode the data was stored
668
673
669 strtable = 'table'
674 strtable = 'table'
670 dsDict['mode'] = 1 # Mode parameters
675 dsDict['mode'] = 1 # Mode parameters
671
676
672 # Three-dimension arrays
677 # Three-dimension arrays
673 if len(arrayDim0) == 3:
678 if len(arrayDim0) == 3:
674 arrayDim[i,1:-1] = numpy.array(arrayDim0)
679 arrayDim[i,1:-1] = numpy.array(arrayDim0)
675 nTables = int(arrayDim[i,2])
680 nTables = int(arrayDim[i,2])
676 dsDict['dsNumber'] = nTables
681 dsDict['dsNumber'] = nTables
677 dsDict['shape'] = arrayDim[i,2:4]
682 dsDict['shape'] = arrayDim[i,2:4]
678 dsDict['nDim'] = 3
683 dsDict['nDim'] = 3
679
684
680 for j in range(nTables):
685 for j in range(nTables):
681 dsDict = dsDict.copy()
686 dsDict = dsDict.copy()
682 dsDict['dsName'] = strtable + str(j)
687 dsDict['dsName'] = strtable + str(j)
683 dsList.append(dsDict)
688 dsList.append(dsDict)
684
689
685 # Two-dimension arrays
690 # Two-dimension arrays
686 elif len(arrayDim0) == 2:
691 elif len(arrayDim0) == 2:
687 arrayDim[i,2:-1] = numpy.array(arrayDim0)
692 arrayDim[i,2:-1] = numpy.array(arrayDim0)
688 nTables = int(arrayDim[i,2])
693 nTables = int(arrayDim[i,2])
689 dsDict['dsNumber'] = nTables
694 dsDict['dsNumber'] = nTables
690 dsDict['shape'] = arrayDim[i,3]
695 dsDict['shape'] = arrayDim[i,3]
691 dsDict['nDim'] = 2
696 dsDict['nDim'] = 2
692
697
693 for j in range(nTables):
698 for j in range(nTables):
694 dsDict = dsDict.copy()
699 dsDict = dsDict.copy()
695 dsDict['dsName'] = strtable + str(j)
700 dsDict['dsName'] = strtable + str(j)
696 dsList.append(dsDict)
701 dsList.append(dsDict)
697
702
698 # One-dimension arrays
703 # One-dimension arrays
699 elif len(arrayDim0) == 1:
704 elif len(arrayDim0) == 1:
700 arrayDim[i,3] = arrayDim0[0]
705 arrayDim[i,3] = arrayDim0[0]
701 dsDict['shape'] = arrayDim0[0]
706 dsDict['shape'] = arrayDim0[0]
702 dsDict['dsNumber'] = 1
707 dsDict['dsNumber'] = 1
703 dsDict['dsName'] = strtable + str(0)
708 dsDict['dsName'] = strtable + str(0)
704 dsDict['nDim'] = 1
709 dsDict['nDim'] = 1
705 dsList.append(dsDict)
710 dsList.append(dsDict)
706
711
707 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
712 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
708 tableList.append(table)
713 tableList.append(table)
709
714
710 # self.arrayDim = arrayDim
715 # self.arrayDim = arrayDim
711 self.dsList = dsList
716 self.dsList = dsList
712 self.tableDim = numpy.array(tableList, dtype = dtype0)
717 self.tableDim = numpy.array(tableList, dtype = dtype0)
713 self.blockIndex = 0
718 self.blockIndex = 0
714
719
715 timeTuple = time.localtime(dataOut.utctime)
720 timeTuple = time.localtime(dataOut.utctime)
716 self.currentDay = timeTuple.tm_yday
721 self.currentDay = timeTuple.tm_yday
717 return 1
722 return 1
718
723
719 def putMetadata(self):
724 def putMetadata(self):
720
725
721 fp = self.createMetadataFile()
726 fp = self.createMetadataFile()
722 self.writeMetadata(fp)
727 self.writeMetadata(fp)
723 fp.close()
728 fp.close()
724 return
729 return
725
730
726 def createMetadataFile(self):
731 def createMetadataFile(self):
727 ext = self.ext
732 ext = self.ext
728 path = self.path
733 path = self.path
729 setFile = self.setFile
734 setFile = self.setFile
730
735
731 timeTuple = time.localtime(self.dataOut.utctime)
736 timeTuple = time.localtime(self.dataOut.utctime)
732
737
733 subfolder = ''
738 subfolder = ''
734 fullpath = os.path.join( path, subfolder )
739 fullpath = os.path.join( path, subfolder )
735
740
736 if not( os.path.exists(fullpath) ):
741 if not( os.path.exists(fullpath) ):
737 os.mkdir(fullpath)
742 os.mkdir(fullpath)
738 setFile = -1 #inicializo mi contador de seteo
743 setFile = -1 #inicializo mi contador de seteo
739
744
740 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
745 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
741 fullpath = os.path.join( path, subfolder )
746 fullpath = os.path.join( path, subfolder )
742
747
743 if not( os.path.exists(fullpath) ):
748 if not( os.path.exists(fullpath) ):
744 os.mkdir(fullpath)
749 os.mkdir(fullpath)
745 setFile = -1 #inicializo mi contador de seteo
750 setFile = -1 #inicializo mi contador de seteo
746
751
747 else:
752 else:
748 filesList = os.listdir( fullpath )
753 filesList = os.listdir( fullpath )
749 filesList = sorted( filesList, key=str.lower )
754 filesList = sorted( filesList, key=str.lower )
750 if len( filesList ) > 0:
755 if len( filesList ) > 0:
751 filesList = [k for k in filesList if 'M' in k]
756 filesList = [k for k in filesList if 'M' in k]
752 filen = filesList[-1]
757 filen = filesList[-1]
753 # el filename debera tener el siguiente formato
758 # el filename debera tener el siguiente formato
754 # 0 1234 567 89A BCDE (hex)
759 # 0 1234 567 89A BCDE (hex)
755 # x YYYY DDD SSS .ext
760 # x YYYY DDD SSS .ext
756 if isNumber( filen[8:11] ):
761 if isNumber( filen[8:11] ):
757 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
762 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
758 else:
763 else:
759 setFile = -1
764 setFile = -1
760 else:
765 else:
761 setFile = -1 #inicializo mi contador de seteo
766 setFile = -1 #inicializo mi contador de seteo
762
767
763 if self.setType is None:
768 if self.setType is None:
764 setFile += 1
769 setFile += 1
765 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
770 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
766 timeTuple.tm_year,
771 timeTuple.tm_year,
767 timeTuple.tm_yday,
772 timeTuple.tm_yday,
768 setFile,
773 setFile,
769 ext )
774 ext )
770 else:
775 else:
771 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
776 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
772 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
777 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
773 timeTuple.tm_year,
778 timeTuple.tm_year,
774 timeTuple.tm_yday,
779 timeTuple.tm_yday,
775 setFile,
780 setFile,
776 ext )
781 ext )
777
782
778 filename = os.path.join( path, subfolder, file )
783 filename = os.path.join( path, subfolder, file )
779 self.metaFile = file
784 self.metaFile = file
780 #Setting HDF5 File
785 #Setting HDF5 File
781 fp = h5py.File(filename,'w')
786 fp = h5py.File(filename,'w')
782
787
783 return fp
788 return fp
784
789
785 def writeMetadata(self, fp):
790 def writeMetadata(self, fp):
786
791
787 grp = fp.create_group("Metadata")
792 grp = fp.create_group("Metadata")
788 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
793 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
789
794
790 for i in range(len(self.metadataList)):
795 for i in range(len(self.metadataList)):
791 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
796 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
792 return
797 return
793
798
794 def timeFlag(self):
799 def timeFlag(self):
795 currentTime = self.dataOut.utctime
800 currentTime = self.dataOut.utctime
796
801
797 if self.lastTime is None:
802 if self.lastTime is None:
798 self.lastTime = currentTime
803 self.lastTime = currentTime
799
804
800 #Day
805 #Day
801 timeTuple = time.localtime(currentTime)
806 timeTuple = time.localtime(currentTime)
802 dataDay = timeTuple.tm_yday
807 dataDay = timeTuple.tm_yday
803
808
804 #Time
809 #Time
805 timeDiff = currentTime - self.lastTime
810 timeDiff = currentTime - self.lastTime
806
811
807 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
812 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
808 if dataDay != self.currentDay:
813 if dataDay != self.currentDay:
809 self.currentDay = dataDay
814 self.currentDay = dataDay
810 return True
815 return True
811 elif timeDiff > 3*60*60:
816 elif timeDiff > 3*60*60:
812 self.lastTime = currentTime
817 self.lastTime = currentTime
813 return True
818 return True
814 else:
819 else:
815 self.lastTime = currentTime
820 self.lastTime = currentTime
816 return False
821 return False
817
822
818 def setNextFile(self):
823 def setNextFile(self):
819
824
820 ext = self.ext
825 ext = self.ext
821 path = self.path
826 path = self.path
822 setFile = self.setFile
827 setFile = self.setFile
823 mode = self.mode
828 mode = self.mode
824
829
825 timeTuple = time.localtime(self.dataOut.utctime)
830 timeTuple = time.localtime(self.dataOut.utctime)
826 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
831 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
827
832
828 fullpath = os.path.join( path, subfolder )
833 fullpath = os.path.join( path, subfolder )
829
834
830 if os.path.exists(fullpath):
835 if os.path.exists(fullpath):
831 filesList = os.listdir( fullpath )
836 filesList = os.listdir( fullpath )
832 filesList = [k for k in filesList if 'D' in k]
837 filesList = [k for k in filesList if 'D' in k]
833 if len( filesList ) > 0:
838 if len( filesList ) > 0:
834 filesList = sorted( filesList, key=str.lower )
839 filesList = sorted( filesList, key=str.lower )
835 filen = filesList[-1]
840 filen = filesList[-1]
836 # el filename debera tener el siguiente formato
841 # el filename debera tener el siguiente formato
837 # 0 1234 567 89A BCDE (hex)
842 # 0 1234 567 89A BCDE (hex)
838 # x YYYY DDD SSS .ext
843 # x YYYY DDD SSS .ext
839 if isNumber( filen[8:11] ):
844 if isNumber( filen[8:11] ):
840 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
845 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
841 else:
846 else:
842 setFile = -1
847 setFile = -1
843 else:
848 else:
844 setFile = -1 #inicializo mi contador de seteo
849 setFile = -1 #inicializo mi contador de seteo
845 else:
850 else:
846 os.makedirs(fullpath)
851 os.makedirs(fullpath)
847 setFile = -1 #inicializo mi contador de seteo
852 setFile = -1 #inicializo mi contador de seteo
848
853
849 if self.setType is None:
854 if None is None:
850 setFile += 1
855 setFile += 1
851 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
856 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
852 timeTuple.tm_year,
857 timeTuple.tm_year,
853 timeTuple.tm_yday,
858 timeTuple.tm_yday,
854 setFile,
859 setFile,
855 ext )
860 ext )
856 else:
861 else:
857 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
862 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
858 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
863 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
859 timeTuple.tm_year,
864 timeTuple.tm_year,
860 timeTuple.tm_yday,
865 timeTuple.tm_yday,
861 setFile,
866 setFile,
862 ext )
867 ext )
863
868
864 filename = os.path.join( path, subfolder, file )
869 filename = os.path.join( path, subfolder, file )
865
870
866 #Setting HDF5 File
871 #Setting HDF5 File
867 fp = h5py.File(filename,'w')
872 fp = h5py.File(filename,'w')
868 #write metadata
873 #write metadata
869 self.writeMetadata(fp)
874 self.writeMetadata(fp)
870 #Write data
875 #Write data
871 grp = fp.create_group("Data")
876 grp = fp.create_group("Data")
872 # grp.attrs['metadata'] = self.metaFile
877 # grp.attrs['metadata'] = self.metaFile
873
878
874 # grp.attrs['blocksPerFile'] = 0
879 # grp.attrs['blocksPerFile'] = 0
875 ds = []
880 ds = []
876 data = []
881 data = []
877 dsList = self.dsList
882 dsList = self.dsList
878 i = 0
883 i = 0
879 while i < len(dsList):
884 while i < len(dsList):
880 dsInfo = dsList[i]
885 dsInfo = dsList[i]
881 #One-dimension data
886 #One-dimension data
882 if dsInfo['mode'] == 0:
887 if dsInfo['mode'] == 0:
883 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
888 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
884 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
889 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
885 ds.append(ds0)
890 ds.append(ds0)
886 data.append([])
891 data.append([])
887 i += 1
892 i += 1
888 continue
893 continue
889 # nDimsForDs.append(nDims[i])
894 # nDimsForDs.append(nDims[i])
890
895
891 elif dsInfo['mode'] == 2:
896 elif dsInfo['mode'] == 2:
892 grp0 = grp.create_group(dsInfo['variable'])
897 grp0 = grp.create_group(dsInfo['variable'])
893 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
898 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
894 ds.append(ds0)
899 ds.append(ds0)
895 data.append([])
900 data.append([])
896 i += 1
901 i += 1
897 continue
902 continue
898
903
899 elif dsInfo['mode'] == 1:
904 elif dsInfo['mode'] == 1:
900 grp0 = grp.create_group(dsInfo['variable'])
905 grp0 = grp.create_group(dsInfo['variable'])
901
906
902 for j in range(dsInfo['dsNumber']):
907 for j in range(dsInfo['dsNumber']):
903 dsInfo = dsList[i]
908 dsInfo = dsList[i]
904 tableName = dsInfo['dsName']
909 tableName = dsInfo['dsName']
905 shape = int(dsInfo['shape'])
910
906
911
907 if dsInfo['nDim'] == 3:
912 if dsInfo['nDim'] == 3:
913 shape = dsInfo['shape'].astype(int)
908 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
914 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
909 else:
915 else:
916 shape = int(dsInfo['shape'])
910 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
917 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
911
918
912 ds.append(ds0)
919 ds.append(ds0)
913 data.append([])
920 data.append([])
914 i += 1
921 i += 1
915 # nDimsForDs.append(nDims[i])
922 # nDimsForDs.append(nDims[i])
916
923
917 fp.flush()
924 fp.flush()
918 fp.close()
925 fp.close()
919
926
920 # self.nDatas = nDatas
927 # self.nDatas = nDatas
921 # self.nDims = nDims
928 # self.nDims = nDims
922 # self.nDimsForDs = nDimsForDs
929 # self.nDimsForDs = nDimsForDs
923 #Saving variables
930 #Saving variables
924 print('Writing the file: %s'%filename)
931 print('Writing the file: %s'%filename)
925 self.filename = filename
932 self.filename = filename
926 # self.fp = fp
933 # self.fp = fp
927 # self.grp = grp
934 # self.grp = grp
928 # self.grp.attrs.modify('nRecords', 1)
935 # self.grp.attrs.modify('nRecords', 1)
929 self.ds = ds
936 self.ds = ds
930 self.data = data
937 self.data = data
931 # self.setFile = setFile
938 # self.setFile = setFile
932 self.firsttime = True
939 self.firsttime = True
933 self.blockIndex = 0
940 self.blockIndex = 0
934 return
941 return
935
942
936 def putData(self):
943 def putData(self):
937
944
938 if self.blockIndex == self.blocksPerFile or self.timeFlag():
945 if self.blockIndex == self.blocksPerFile or self.timeFlag():
939 self.setNextFile()
946 self.setNextFile()
940
947
941 # if not self.firsttime:
948 # if not self.firsttime:
942 self.readBlock()
949 self.readBlock()
943 self.setBlock() #Prepare data to be written
950 self.setBlock() #Prepare data to be written
944 self.writeBlock() #Write data
951 self.writeBlock() #Write data
945
952
946 return
953 return
947
954
948 def readBlock(self):
955 def readBlock(self):
949
956
950 '''
957 '''
951 data Array configured
958 data Array configured
952
959
953
960
954 self.data
961 self.data
955 '''
962 '''
956 dsList = self.dsList
963 dsList = self.dsList
957 ds = self.ds
964 ds = self.ds
958 #Setting HDF5 File
965 #Setting HDF5 File
959 fp = h5py.File(self.filename,'r+')
966 fp = h5py.File(self.filename,'r+')
960 grp = fp["Data"]
967 grp = fp["Data"]
961 ind = 0
968 ind = 0
962
969
963 # grp.attrs['blocksPerFile'] = 0
970 # grp.attrs['blocksPerFile'] = 0
964 while ind < len(dsList):
971 while ind < len(dsList):
965 dsInfo = dsList[ind]
972 dsInfo = dsList[ind]
966
973
967 if dsInfo['mode'] == 0:
974 if dsInfo['mode'] == 0:
968 ds0 = grp[dsInfo['variable']]
975 ds0 = grp[dsInfo['variable']]
969 ds[ind] = ds0
976 ds[ind] = ds0
970 ind += 1
977 ind += 1
971 else:
978 else:
972
979
973 grp0 = grp[dsInfo['variable']]
980 grp0 = grp[dsInfo['variable']]
974
981
975 for j in range(dsInfo['dsNumber']):
982 for j in range(dsInfo['dsNumber']):
976 dsInfo = dsList[ind]
983 dsInfo = dsList[ind]
977 ds0 = grp0[dsInfo['dsName']]
984 ds0 = grp0[dsInfo['dsName']]
978 ds[ind] = ds0
985 ds[ind] = ds0
979 ind += 1
986 ind += 1
980
987
981 self.fp = fp
988 self.fp = fp
982 self.grp = grp
989 self.grp = grp
983 self.ds = ds
990 self.ds = ds
984
991
985 return
992 return
986
993
987 def setBlock(self):
994 def setBlock(self):
988 '''
995 '''
989 data Array configured
996 data Array configured
990
997
991
998
992 self.data
999 self.data
993 '''
1000 '''
994 #Creating Arrays
1001 #Creating Arrays
995 dsList = self.dsList
1002 dsList = self.dsList
996 data = self.data
1003 data = self.data
997 ind = 0
1004 ind = 0
998
1005
999 while ind < len(dsList):
1006 while ind < len(dsList):
1000 dsInfo = dsList[ind]
1007 dsInfo = dsList[ind]
1001 dataAux = getattr(self.dataOut, dsInfo['variable'])
1008 dataAux = getattr(self.dataOut, dsInfo['variable'])
1002
1009
1003 mode = dsInfo['mode']
1010 mode = dsInfo['mode']
1004 nDim = dsInfo['nDim']
1011 nDim = dsInfo['nDim']
1005
1012
1006 if mode == 0 or mode == 2 or nDim == 1:
1013 if mode == 0 or mode == 2 or nDim == 1:
1007 data[ind] = dataAux
1014 data[ind] = dataAux
1008 ind += 1
1015 ind += 1
1009 # elif nDim == 1:
1016 # elif nDim == 1:
1010 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1017 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1011 # ind += 1
1018 # ind += 1
1012 elif nDim == 2:
1019 elif nDim == 2:
1013 for j in range(dsInfo['dsNumber']):
1020 for j in range(dsInfo['dsNumber']):
1014 data[ind] = dataAux[j,:]
1021 data[ind] = dataAux[j,:]
1015 ind += 1
1022 ind += 1
1016 elif nDim == 3:
1023 elif nDim == 3:
1017 for j in range(dsInfo['dsNumber']):
1024 for j in range(dsInfo['dsNumber']):
1018 data[ind] = dataAux[:,j,:]
1025 data[ind] = dataAux[:,j,:]
1019 ind += 1
1026 ind += 1
1020
1027
1021 self.data = data
1028 self.data = data
1022 return
1029 return
1023
1030
1024 def writeBlock(self):
1031 def writeBlock(self):
1025 '''
1032 '''
1026 Saves the block in the HDF5 file
1033 Saves the block in the HDF5 file
1027 '''
1034 '''
1028 dsList = self.dsList
1035 dsList = self.dsList
1029
1036
1030 for i in range(len(self.ds)):
1037 for i in range(len(self.ds)):
1031 dsInfo = dsList[i]
1038 dsInfo = dsList[i]
1032 nDim = dsInfo['nDim']
1039 nDim = dsInfo['nDim']
1033 mode = dsInfo['mode']
1040 mode = dsInfo['mode']
1034
1041
1035 # First time
1042 # First time
1036 if self.firsttime:
1043 if self.firsttime:
1037 # self.ds[i].resize(self.data[i].shape)
1044 # self.ds[i].resize(self.data[i].shape)
1038 # self.ds[i][self.blockIndex,:] = self.data[i]
1045 # self.ds[i][self.blockIndex,:] = self.data[i]
1039 if type(self.data[i]) == numpy.ndarray:
1046 if type(self.data[i]) == numpy.ndarray:
1040
1047
1041 if nDim == 3:
1048 if nDim == 3:
1042 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1049 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1043 self.ds[i].resize(self.data[i].shape)
1050 self.ds[i].resize(self.data[i].shape)
1044 if mode == 2:
1051 if mode == 2:
1045 self.ds[i].resize(self.data[i].shape)
1052 self.ds[i].resize(self.data[i].shape)
1046 self.ds[i][:] = self.data[i]
1053 self.ds[i][:] = self.data[i]
1047 else:
1054 else:
1048
1055
1049 # From second time
1056 # From second time
1050 # Meteors!
1057 # Meteors!
1051 if mode == 2:
1058 if mode == 2:
1052 dataShape = self.data[i].shape
1059 dataShape = self.data[i].shape
1053 dsShape = self.ds[i].shape
1060 dsShape = self.ds[i].shape
1054 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1061 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1055 self.ds[i][dsShape[0]:,:] = self.data[i]
1062 self.ds[i][dsShape[0]:,:] = self.data[i]
1056 # No dimension
1063 # No dimension
1057 elif mode == 0:
1064 elif mode == 0:
1058 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1065 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1059 self.ds[i][0,-1] = self.data[i]
1066 self.ds[i][0,-1] = self.data[i]
1060 # One dimension
1067 # One dimension
1061 elif nDim == 1:
1068 elif nDim == 1:
1062 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1069 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1063 self.ds[i][-1,:] = self.data[i]
1070 self.ds[i][-1,:] = self.data[i]
1064 # Two dimension
1071 # Two dimension
1065 elif nDim == 2:
1072 elif nDim == 2:
1066 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1073 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1067 self.ds[i][self.blockIndex,:] = self.data[i]
1074 self.ds[i][self.blockIndex,:] = self.data[i]
1068 # Three dimensions
1075 # Three dimensions
1069 elif nDim == 3:
1076 elif nDim == 3:
1070 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1077 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1071 self.ds[i][:,:,-1] = self.data[i]
1078 self.ds[i][:,:,-1] = self.data[i]
1072
1079
1073 self.firsttime = False
1080 self.firsttime = False
1074 self.blockIndex += 1
1081 self.blockIndex += 1
1075
1082
1076 #Close to save changes
1083 #Close to save changes
1077 self.fp.flush()
1084 self.fp.flush()
1078 self.fp.close()
1085 self.fp.close()
1079 return
1086 return
1080
1087
1081 def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
1088 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
1082
1089
1083 if not(self.isConfig):
1090 if not(self.isConfig):
1084 flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
1091 flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
1085 metadataList=metadataList, dataList=dataList, mode=mode, **kwargs)
1092 metadataList=metadataList, dataList=dataList, mode=mode, **kwargs)
1086
1093
1087 if not(flagdata):
1094 if not(flagdata):
1088 return
1095 return
1089
1096
1090 self.isConfig = True
1097 self.isConfig = True
1091 # self.putMetadata()
1098 # self.putMetadata()
1092 self.setNextFile()
1099 self.setNextFile()
1093
1100
1094 self.putData()
1101 self.putData()
1095 return No newline at end of file
1102 return
1103 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,680 +1,679
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import numpy
6 import numpy
7
7
8 from schainpy.model.io.jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
8 from schainpy.model.io.jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
10 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
11 from schainpy.model.data.jrodata import Spectra
11 from schainpy.model.data.jrodata import Spectra
12 from schainpy.utils import log
12
13
13 @MPDecorator
14 @MPDecorator
14 class SpectraReader(JRODataReader, ProcessingUnit):
15 class SpectraReader(JRODataReader, ProcessingUnit):
15 """
16 """
16 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
17 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
17 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
18 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
18 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
19 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
19
20
20 paresCanalesIguales * alturas * perfiles (Self Spectra)
21 paresCanalesIguales * alturas * perfiles (Self Spectra)
21 paresCanalesDiferentes * alturas * perfiles (Cross Spectra)
22 paresCanalesDiferentes * alturas * perfiles (Cross Spectra)
22 canales * alturas (DC Channels)
23 canales * alturas (DC Channels)
23
24
24 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
25 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
25 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
26 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
26 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
27 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
27 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
28 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
28
29
29 Example:
30 Example:
30 dpath = "/home/myuser/data"
31 dpath = "/home/myuser/data"
31
32
32 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
33 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
33
34
34 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
35 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
35
36
36 readerObj = SpectraReader()
37 readerObj = SpectraReader()
37
38
38 readerObj.setup(dpath, startTime, endTime)
39 readerObj.setup(dpath, startTime, endTime)
39
40
40 while(True):
41 while(True):
41
42
42 readerObj.getData()
43 readerObj.getData()
43
44
44 print readerObj.data_spc
45 print readerObj.data_spc
45
46
46 print readerObj.data_cspc
47 print readerObj.data_cspc
47
48
48 print readerObj.data_dc
49 print readerObj.data_dc
49
50
50 if readerObj.flagNoMoreFiles:
51 if readerObj.flagNoMoreFiles:
51 break
52 break
52
53
53 """
54 """
54
55
55 pts2read_SelfSpectra = 0
56 pts2read_SelfSpectra = 0
56
57
57 pts2read_CrossSpectra = 0
58 pts2read_CrossSpectra = 0
58
59
59 pts2read_DCchannels = 0
60 pts2read_DCchannels = 0
60
61
61 ext = ".pdata"
62 ext = ".pdata"
62
63
63 optchar = "P"
64 optchar = "P"
64
65
65 dataOut = None
66 dataOut = None
66
67
67 nRdChannels = None
68 nRdChannels = None
68
69
69 nRdPairs = None
70 nRdPairs = None
70
71
71 rdPairList = []
72 rdPairList = []
72
73
73 def __init__(self):#, **kwargs):
74 def __init__(self):#, **kwargs):
74 """
75 """
75 Inicializador de la clase SpectraReader para la lectura de datos de espectros.
76 Inicializador de la clase SpectraReader para la lectura de datos de espectros.
76
77
77 Inputs:
78 Inputs:
78 dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para
79 dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para
79 almacenar un perfil de datos cada vez que se haga un requerimiento
80 almacenar un perfil de datos cada vez que se haga un requerimiento
80 (getData). El perfil sera obtenido a partir del buffer de datos,
81 (getData). El perfil sera obtenido a partir del buffer de datos,
81 si el buffer esta vacio se hara un nuevo proceso de lectura de un
82 si el buffer esta vacio se hara un nuevo proceso de lectura de un
82 bloque de datos.
83 bloque de datos.
83 Si este parametro no es pasado se creara uno internamente.
84 Si este parametro no es pasado se creara uno internamente.
84
85
85 Affected:
86 Affected:
86 self.dataOut
87 self.dataOut
87
88
88 Return : None
89 Return : None
89 """
90 """
90
91
91 #Eliminar de la base la herencia
92 #Eliminar de la base la herencia
92 ProcessingUnit.__init__(self)#, **kwargs)
93 ProcessingUnit.__init__(self)#, **kwargs)
93
94
94 # self.isConfig = False
95 # self.isConfig = False
95
96
96 self.pts2read_SelfSpectra = 0
97 self.pts2read_SelfSpectra = 0
97
98
98 self.pts2read_CrossSpectra = 0
99 self.pts2read_CrossSpectra = 0
99
100
100 self.pts2read_DCchannels = 0
101 self.pts2read_DCchannels = 0
101
102
102 self.datablock = None
103 self.datablock = None
103
104
104 self.utc = None
105 self.utc = None
105
106
106 self.ext = ".pdata"
107 self.ext = ".pdata"
107
108
108 self.optchar = "P"
109 self.optchar = "P"
109
110
110 self.basicHeaderObj = BasicHeader(LOCALTIME)
111 self.basicHeaderObj = BasicHeader(LOCALTIME)
111
112
112 self.systemHeaderObj = SystemHeader()
113 self.systemHeaderObj = SystemHeader()
113
114
114 self.radarControllerHeaderObj = RadarControllerHeader()
115 self.radarControllerHeaderObj = RadarControllerHeader()
115
116
116 self.processingHeaderObj = ProcessingHeader()
117 self.processingHeaderObj = ProcessingHeader()
117
118
118 self.online = 0
119 self.online = 0
119
120
120 self.fp = None
121 self.fp = None
121
122
122 self.idFile = None
123 self.idFile = None
123
124
124 self.dtype = None
125 self.dtype = None
125
126
126 self.fileSizeByHeader = None
127 self.fileSizeByHeader = None
127
128
128 self.filenameList = []
129 self.filenameList = []
129
130
130 self.filename = None
131 self.filename = None
131
132
132 self.fileSize = None
133 self.fileSize = None
133
134
134 self.firstHeaderSize = 0
135 self.firstHeaderSize = 0
135
136
136 self.basicHeaderSize = 24
137 self.basicHeaderSize = 24
137
138
138 self.pathList = []
139 self.pathList = []
139
140
140 self.lastUTTime = 0
141 self.lastUTTime = 0
141
142
142 self.maxTimeStep = 30
143 self.maxTimeStep = 30
143
144
144 self.flagNoMoreFiles = 0
145 self.flagNoMoreFiles = 0
145
146
146 self.set = 0
147 self.set = 0
147
148
148 self.path = None
149 self.path = None
149
150
150 self.delay = 60 #seconds
151 self.delay = 60 #seconds
151
152
152 self.nTries = 3 #quantity tries
153 self.nTries = 3 #quantity tries
153
154
154 self.nFiles = 3 #number of files for searching
155 self.nFiles = 3 #number of files for searching
155
156
156 self.nReadBlocks = 0
157 self.nReadBlocks = 0
157
158
158 self.flagIsNewFile = 1
159 self.flagIsNewFile = 1
159
160
160 self.__isFirstTimeOnline = 1
161 self.__isFirstTimeOnline = 1
161
162
162 # self.ippSeconds = 0
163 # self.ippSeconds = 0
163
164
164 self.flagDiscontinuousBlock = 0
165 self.flagDiscontinuousBlock = 0
165
166
166 self.flagIsNewBlock = 0
167 self.flagIsNewBlock = 0
167
168
168 self.nTotalBlocks = 0
169 self.nTotalBlocks = 0
169
170
170 self.blocksize = 0
171 self.blocksize = 0
171
172
172 self.dataOut = self.createObjByDefault()
173 self.dataOut = self.createObjByDefault()
173
174
174 self.profileIndex = 1 #Always
175 self.profileIndex = 1 #Always
175
176
176
177
177 def createObjByDefault(self):
178 def createObjByDefault(self):
178
179
179 dataObj = Spectra()
180 dataObj = Spectra()
180
181
181 return dataObj
182 return dataObj
182
183
183 def __hasNotDataInBuffer(self):
184 def __hasNotDataInBuffer(self):
184 return 1
185 return 1
185
186
186
187
187 def getBlockDimension(self):
188 def getBlockDimension(self):
188 """
189 """
189 Obtiene la cantidad de puntos a leer por cada bloque de datos
190 Obtiene la cantidad de puntos a leer por cada bloque de datos
190
191
191 Affected:
192 Affected:
192 self.nRdChannels
193 self.nRdChannels
193 self.nRdPairs
194 self.nRdPairs
194 self.pts2read_SelfSpectra
195 self.pts2read_SelfSpectra
195 self.pts2read_CrossSpectra
196 self.pts2read_CrossSpectra
196 self.pts2read_DCchannels
197 self.pts2read_DCchannels
197 self.blocksize
198 self.blocksize
198 self.dataOut.nChannels
199 self.dataOut.nChannels
199 self.dataOut.nPairs
200 self.dataOut.nPairs
200
201
201 Return:
202 Return:
202 None
203 None
203 """
204 """
204 self.nRdChannels = 0
205 self.nRdChannels = 0
205 self.nRdPairs = 0
206 self.nRdPairs = 0
206 self.rdPairList = []
207 self.rdPairList = []
207
208
208 for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
209 for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
209 if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
210 if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
210 self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
211 self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
211 else:
212 else:
212 self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
213 self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
213 self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
214 self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
214
215
215 pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
216 pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
216
217
217 self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
218 self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
218 self.blocksize = self.pts2read_SelfSpectra
219 self.blocksize = self.pts2read_SelfSpectra
219
220
220 if self.processingHeaderObj.flag_cspc:
221 if self.processingHeaderObj.flag_cspc:
221 self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
222 self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
222 self.blocksize += self.pts2read_CrossSpectra
223 self.blocksize += self.pts2read_CrossSpectra
223
224
224 if self.processingHeaderObj.flag_dc:
225 if self.processingHeaderObj.flag_dc:
225 self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
226 self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
226 self.blocksize += self.pts2read_DCchannels
227 self.blocksize += self.pts2read_DCchannels
227
228
228 # self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
229 # self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
229
230
230
231
231 def readBlock(self):
232 def readBlock(self):
232 """
233 """
233 Lee el bloque de datos desde la posicion actual del puntero del archivo
234 Lee el bloque de datos desde la posicion actual del puntero del archivo
234 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
235 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
235 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
236 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
236 es seteado a 0
237 es seteado a 0
237
238
238 Return: None
239 Return: None
239
240
240 Variables afectadas:
241 Variables afectadas:
241
242
242 self.flagIsNewFile
243 self.flagIsNewFile
243 self.flagIsNewBlock
244 self.flagIsNewBlock
244 self.nTotalBlocks
245 self.nTotalBlocks
245 self.data_spc
246 self.data_spc
246 self.data_cspc
247 self.data_cspc
247 self.data_dc
248 self.data_dc
248
249
249 Exceptions:
250 Exceptions:
250 Si un bloque leido no es un bloque valido
251 Si un bloque leido no es un bloque valido
251 """
252 """
252 blockOk_flag = False
253 blockOk_flag = False
253 fpointer = self.fp.tell()
254 fpointer = self.fp.tell()
254
255
255 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
256 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
256 spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
257 spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
257
258
258 if self.processingHeaderObj.flag_cspc:
259 if self.processingHeaderObj.flag_cspc:
259 cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
260 cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
260 cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
261 cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
261
262
262 if self.processingHeaderObj.flag_dc:
263 if self.processingHeaderObj.flag_dc:
263 dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
264 dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
264 dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
265 dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
265
266
266
267
267 if not self.processingHeaderObj.shif_fft:
268 if not self.processingHeaderObj.shif_fft:
268 #desplaza a la derecha en el eje 2 determinadas posiciones
269 #desplaza a la derecha en el eje 2 determinadas posiciones
269 shift = int(self.processingHeaderObj.profilesPerBlock/2)
270 shift = int(self.processingHeaderObj.profilesPerBlock/2)
270 spc = numpy.roll( spc, shift , axis=2 )
271 spc = numpy.roll( spc, shift , axis=2 )
271
272
272 if self.processingHeaderObj.flag_cspc:
273 if self.processingHeaderObj.flag_cspc:
273 #desplaza a la derecha en el eje 2 determinadas posiciones
274 #desplaza a la derecha en el eje 2 determinadas posiciones
274 cspc = numpy.roll( cspc, shift, axis=2 )
275 cspc = numpy.roll( cspc, shift, axis=2 )
275
276
276 #Dimensions : nChannels, nProfiles, nSamples
277 #Dimensions : nChannels, nProfiles, nSamples
277 spc = numpy.transpose( spc, (0,2,1) )
278 spc = numpy.transpose( spc, (0,2,1) )
278 self.data_spc = spc
279 self.data_spc = spc
279
280
280 if self.processingHeaderObj.flag_cspc:
281 if self.processingHeaderObj.flag_cspc:
281 cspc = numpy.transpose( cspc, (0,2,1) )
282 cspc = numpy.transpose( cspc, (0,2,1) )
282 self.data_cspc = cspc['real'] + cspc['imag']*1j
283 self.data_cspc = cspc['real'] + cspc['imag']*1j
283 else:
284 else:
284 self.data_cspc = None
285 self.data_cspc = None
285
286
286 if self.processingHeaderObj.flag_dc:
287 if self.processingHeaderObj.flag_dc:
287 self.data_dc = dc['real'] + dc['imag']*1j
288 self.data_dc = dc['real'] + dc['imag']*1j
288 else:
289 else:
289 self.data_dc = None
290 self.data_dc = None
290
291
291 self.flagIsNewFile = 0
292 self.flagIsNewFile = 0
292 self.flagIsNewBlock = 1
293 self.flagIsNewBlock = 1
293
294
294 self.nTotalBlocks += 1
295 self.nTotalBlocks += 1
295 self.nReadBlocks += 1
296 self.nReadBlocks += 1
296
297
297 return 1
298 return 1
298
299
299 def getFirstHeader(self):
300 def getFirstHeader(self):
300
301
301 self.getBasicHeader()
302 self.getBasicHeader()
302
303
303 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
304 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
304
305
305 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
306 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
306
307
307 # self.dataOut.ippSeconds = self.ippSeconds
308 # self.dataOut.ippSeconds = self.ippSeconds
308
309
309 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.processingHeaderObj.profilesPerBlock
310 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.processingHeaderObj.profilesPerBlock
310
311
311 self.dataOut.dtype = self.dtype
312 self.dataOut.dtype = self.dtype
312
313
313 # self.dataOut.nPairs = self.nPairs
314 # self.dataOut.nPairs = self.nPairs
314
315
315 self.dataOut.pairsList = self.rdPairList
316 self.dataOut.pairsList = self.rdPairList
316
317
317 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
318 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
318
319
319 self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
320 self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
320
321
321 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
322 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
322
323
323 self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
324 self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
324
325
325 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
326 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
326
327
327 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
328 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
328
329
329 self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
330 self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
330
331
331 self.dataOut.flagShiftFFT = True #Data is always shifted
332 self.dataOut.flagShiftFFT = True #Data is always shifted
332
333
333 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
334 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
334
335
335 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data esta sin flip
336 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data esta sin flip
336
337
337 def getData(self):
338 def getData(self):
338 """
339 """
339 First method to execute before "RUN" is called.
340 First method to execute before "RUN" is called.
340
341
341 Copia el buffer de lectura a la clase "Spectra",
342 Copia el buffer de lectura a la clase "Spectra",
342 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
343 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
343 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
344 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
344
345
345 Return:
346 Return:
346 0 : Si no hay mas archivos disponibles
347 0 : Si no hay mas archivos disponibles
347 1 : Si hizo una buena copia del buffer
348 1 : Si hizo una buena copia del buffer
348
349
349 Affected:
350 Affected:
350 self.dataOut
351 self.dataOut
351
352
352 self.flagDiscontinuousBlock
353 self.flagDiscontinuousBlock
353 self.flagIsNewBlock
354 self.flagIsNewBlock
354 """
355 """
355
356
356 if self.flagNoMoreFiles:
357 if self.flagNoMoreFiles:
357 self.dataOut.flagNoData = True
358 self.dataOut.flagNoData = True
358 print('Process finished')
359 print('Process finished')
359 return 0
360 return 0
360
361
361 self.flagDiscontinuousBlock = 0
362 self.flagDiscontinuousBlock = 0
362 self.flagIsNewBlock = 0
363 self.flagIsNewBlock = 0
363
364
364 if self.__hasNotDataInBuffer():
365 if self.__hasNotDataInBuffer():
365
366
366 if not( self.readNextBlock() ):
367 if not( self.readNextBlock() ):
367 self.dataOut.flagNoData = True
368 self.dataOut.flagNoData = True
368 return 0
369 return 0
369
370
370 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
371 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
371
372
372 if self.data_spc is None:
373 if self.data_spc is None:
373 self.dataOut.flagNoData = True
374 self.dataOut.flagNoData = True
374 return 0
375 return 0
375
376
376 self.getBasicHeader()
377 self.getBasicHeader()
377
378
378 self.getFirstHeader()
379 self.getFirstHeader()
379
380
380 self.dataOut.data_spc = self.data_spc
381 self.dataOut.data_spc = self.data_spc
381
382
382 self.dataOut.data_cspc = self.data_cspc
383 self.dataOut.data_cspc = self.data_cspc
383
384
384 self.dataOut.data_dc = self.data_dc
385 self.dataOut.data_dc = self.data_dc
385
386
386 self.dataOut.flagNoData = False
387 self.dataOut.flagNoData = False
387
388
388 self.dataOut.realtime = self.online
389 self.dataOut.realtime = self.online
389
390
390 return self.dataOut.data_spc
391 return self.dataOut.data_spc
391
392 @MPDecorator
392 class SpectraWriter(JRODataWriter, Operation):
393 class SpectraWriter(JRODataWriter, Operation):
393
394
394 """
395 """
395 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
396 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
396 de los datos siempre se realiza por bloques.
397 de los datos siempre se realiza por bloques.
397 """
398 """
398
399
399 ext = ".pdata"
400 ext = ".pdata"
400
401
401 optchar = "P"
402 optchar = "P"
402
403
403 shape_spc_Buffer = None
404 shape_spc_Buffer = None
404
405
405 shape_cspc_Buffer = None
406 shape_cspc_Buffer = None
406
407
407 shape_dc_Buffer = None
408 shape_dc_Buffer = None
408
409
409 data_spc = None
410 data_spc = None
410
411
411 data_cspc = None
412 data_cspc = None
412
413
413 data_dc = None
414 data_dc = None
414
415
415 # dataOut = None
416 # dataOut = None
416
417
417 def __init__(self, **kwargs):
418 def __init__(self):#, **kwargs):
418 """
419 """
419 Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
420 Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
420
421
421 Affected:
422 Affected:
422 self.dataOut
423 self.dataOut
423 self.basicHeaderObj
424 self.basicHeaderObj
424 self.systemHeaderObj
425 self.systemHeaderObj
425 self.radarControllerHeaderObj
426 self.radarControllerHeaderObj
426 self.processingHeaderObj
427 self.processingHeaderObj
427
428
428 Return: None
429 Return: None
429 """
430 """
430
431
431 Operation.__init__(self, **kwargs)
432 Operation.__init__(self)#, **kwargs)
432
433
433 self.isConfig = False
434 #self.isConfig = False
434
435
435 self.nTotalBlocks = 0
436 self.nTotalBlocks = 0
436
437
437 self.data_spc = None
438 self.data_spc = None
438
439
439 self.data_cspc = None
440 self.data_cspc = None
440
441
441 self.data_dc = None
442 self.data_dc = None
442
443
443 self.fp = None
444 self.fp = None
444
445
445 self.flagIsNewFile = 1
446 self.flagIsNewFile = 1
446
447
447 self.nTotalBlocks = 0
448 self.nTotalBlocks = 0
448
449
449 self.flagIsNewBlock = 0
450 self.flagIsNewBlock = 0
450
451
451 self.setFile = None
452 self.setFile = None
452
453
453 self.dtype = None
454 self.dtype = None
454
455
455 self.path = None
456 self.path = None
456
457
457 self.noMoreFiles = 0
458 self.noMoreFiles = 0
458
459
459 self.filename = None
460 self.filename = None
460
461
461 self.basicHeaderObj = BasicHeader(LOCALTIME)
462 self.basicHeaderObj = BasicHeader(LOCALTIME)
462
463
463 self.systemHeaderObj = SystemHeader()
464 self.systemHeaderObj = SystemHeader()
464
465
465 self.radarControllerHeaderObj = RadarControllerHeader()
466 self.radarControllerHeaderObj = RadarControllerHeader()
466
467
467 self.processingHeaderObj = ProcessingHeader()
468 self.processingHeaderObj = ProcessingHeader()
468
469
469
470
470 def hasAllDataInBuffer(self):
471 def hasAllDataInBuffer(self):
471 return 1
472 return 1
472
473
473
474
474 def setBlockDimension(self):
475 def setBlockDimension(self):
475 """
476 """
476 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
477 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
477
478
478 Affected:
479 Affected:
479 self.shape_spc_Buffer
480 self.shape_spc_Buffer
480 self.shape_cspc_Buffer
481 self.shape_cspc_Buffer
481 self.shape_dc_Buffer
482 self.shape_dc_Buffer
482
483
483 Return: None
484 Return: None
484 """
485 """
485 self.shape_spc_Buffer = (self.dataOut.nChannels,
486 self.shape_spc_Buffer = (self.dataOut.nChannels,
486 self.processingHeaderObj.nHeights,
487 self.processingHeaderObj.nHeights,
487 self.processingHeaderObj.profilesPerBlock)
488 self.processingHeaderObj.profilesPerBlock)
488
489
489 self.shape_cspc_Buffer = (self.dataOut.nPairs,
490 self.shape_cspc_Buffer = (self.dataOut.nPairs,
490 self.processingHeaderObj.nHeights,
491 self.processingHeaderObj.nHeights,
491 self.processingHeaderObj.profilesPerBlock)
492 self.processingHeaderObj.profilesPerBlock)
492
493
493 self.shape_dc_Buffer = (self.dataOut.nChannels,
494 self.shape_dc_Buffer = (self.dataOut.nChannels,
494 self.processingHeaderObj.nHeights)
495 self.processingHeaderObj.nHeights)
495
496
496
497
497 def writeBlock(self):
498 def writeBlock(self):
498 """
499 """
499 Escribe el buffer en el file designado
500 Escribe el buffer en el file designado
500
501
501 Affected:
502 Affected:
502 self.data_spc
503 self.data_spc
503 self.data_cspc
504 self.data_cspc
504 self.data_dc
505 self.data_dc
505 self.flagIsNewFile
506 self.flagIsNewFile
506 self.flagIsNewBlock
507 self.flagIsNewBlock
507 self.nTotalBlocks
508 self.nTotalBlocks
508 self.nWriteBlocks
509 self.nWriteBlocks
509
510
510 Return: None
511 Return: None
511 """
512 """
512
513
513 spc = numpy.transpose( self.data_spc, (0,2,1) )
514 spc = numpy.transpose( self.data_spc, (0,2,1) )
514 if not self.processingHeaderObj.shif_fft:
515 if not self.processingHeaderObj.shif_fft:
515 spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
516 spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
516 data = spc.reshape((-1))
517 data = spc.reshape((-1))
517 data = data.astype(self.dtype[0])
518 data = data.astype(self.dtype[0])
518 data.tofile(self.fp)
519 data.tofile(self.fp)
519
520
520 if self.data_cspc is not None:
521 if self.data_cspc is not None:
521 data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )
522 data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )
522 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
523 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
523 if not self.processingHeaderObj.shif_fft:
524 if not self.processingHeaderObj.shif_fft:
524 cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
525 cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
525 data['real'] = cspc.real
526 data['real'] = cspc.real
526 data['imag'] = cspc.imag
527 data['imag'] = cspc.imag
527 data = data.reshape((-1))
528 data = data.reshape((-1))
528 data.tofile(self.fp)
529 data.tofile(self.fp)
529
530
530 if self.data_dc is not None:
531 if self.data_dc is not None:
531 data = numpy.zeros( self.shape_dc_Buffer, self.dtype )
532 data = numpy.zeros( self.shape_dc_Buffer, self.dtype )
532 dc = self.data_dc
533 dc = self.data_dc
533 data['real'] = dc.real
534 data['real'] = dc.real
534 data['imag'] = dc.imag
535 data['imag'] = dc.imag
535 data = data.reshape((-1))
536 data = data.reshape((-1))
536 data.tofile(self.fp)
537 data.tofile(self.fp)
537
538
538 # self.data_spc.fill(0)
539 # self.data_spc.fill(0)
539 #
540 #
540 # if self.data_dc is not None:
541 # if self.data_dc is not None:
541 # self.data_dc.fill(0)
542 # self.data_dc.fill(0)
542 #
543 #
543 # if self.data_cspc is not None:
544 # if self.data_cspc is not None:
544 # self.data_cspc.fill(0)
545 # self.data_cspc.fill(0)
545
546
546 self.flagIsNewFile = 0
547 self.flagIsNewFile = 0
547 self.flagIsNewBlock = 1
548 self.flagIsNewBlock = 1
548 self.nTotalBlocks += 1
549 self.nTotalBlocks += 1
549 self.nWriteBlocks += 1
550 self.nWriteBlocks += 1
550 self.blockIndex += 1
551 self.blockIndex += 1
551
552
552 # print "[Writing] Block = %d04" %self.blockIndex
553 # print "[Writing] Block = %d04" %self.blockIndex
553
554
554 def putData(self):
555 def putData(self):
555 """
556 """
556 Setea un bloque de datos y luego los escribe en un file
557 Setea un bloque de datos y luego los escribe en un file
557
558
558 Affected:
559 Affected:
559 self.data_spc
560 self.data_spc
560 self.data_cspc
561 self.data_cspc
561 self.data_dc
562 self.data_dc
562
563
563 Return:
564 Return:
564 0 : Si no hay data o no hay mas files que puedan escribirse
565 0 : Si no hay data o no hay mas files que puedan escribirse
565 1 : Si se escribio la data de un bloque en un file
566 1 : Si se escribio la data de un bloque en un file
566 """
567 """
567
568
568 if self.dataOut.flagNoData:
569 if self.dataOut.flagNoData:
569 return 0
570 return 0
570
571
571 self.flagIsNewBlock = 0
572 self.flagIsNewBlock = 0
572
573
573 if self.dataOut.flagDiscontinuousBlock:
574 if self.dataOut.flagDiscontinuousBlock:
574 self.data_spc.fill(0)
575 self.data_spc.fill(0)
575 if self.dataOut.data_cspc is not None:
576 if self.dataOut.data_cspc is not None:
576 self.data_cspc.fill(0)
577 self.data_cspc.fill(0)
577 if self.dataOut.data_dc is not None:
578 if self.dataOut.data_dc is not None:
578 self.data_dc.fill(0)
579 self.data_dc.fill(0)
579 self.setNextFile()
580 self.setNextFile()
580
581
581 if self.flagIsNewFile == 0:
582 if self.flagIsNewFile == 0:
582 self.setBasicHeader()
583 self.setBasicHeader()
583
584
584 self.data_spc = self.dataOut.data_spc.copy()
585 self.data_spc = self.dataOut.data_spc.copy()
585
586
586 if self.dataOut.data_cspc is not None:
587 if self.dataOut.data_cspc is not None:
587 self.data_cspc = self.dataOut.data_cspc.copy()
588 self.data_cspc = self.dataOut.data_cspc.copy()
588
589
589 if self.dataOut.data_dc is not None:
590 if self.dataOut.data_dc is not None:
590 self.data_dc = self.dataOut.data_dc.copy()
591 self.data_dc = self.dataOut.data_dc.copy()
591
592
592 # #self.processingHeaderObj.dataBlocksPerFile)
593 # #self.processingHeaderObj.dataBlocksPerFile)
593 if self.hasAllDataInBuffer():
594 if self.hasAllDataInBuffer():
594 # self.setFirstHeader()
595 # self.setFirstHeader()
595 self.writeNextBlock()
596 self.writeNextBlock()
596
597
597 return 1
598
599 def __getBlockSize(self):
598 def __getBlockSize(self):
600 '''
599 '''
601 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
600 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
602 '''
601 '''
603
602
604 dtype_width = self.getDtypeWidth()
603 dtype_width = self.getDtypeWidth()
605
604
606 pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
605 pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
607
606
608 pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
607 pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
609 blocksize = (pts2write_SelfSpectra*dtype_width)
608 blocksize = (pts2write_SelfSpectra*dtype_width)
610
609
611 if self.dataOut.data_cspc is not None:
610 if self.dataOut.data_cspc is not None:
612 pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
611 pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
613 blocksize += (pts2write_CrossSpectra*dtype_width*2)
612 blocksize += (pts2write_CrossSpectra*dtype_width*2)
614
613
615 if self.dataOut.data_dc is not None:
614 if self.dataOut.data_dc is not None:
616 pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
615 pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
617 blocksize += (pts2write_DCchannels*dtype_width*2)
616 blocksize += (pts2write_DCchannels*dtype_width*2)
618
617
619 # blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
618 # blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
620
619
621 return blocksize
620 return blocksize
622
621
623 def setFirstHeader(self):
622 def setFirstHeader(self):
624
623
625 """
624 """
626 Obtiene una copia del First Header
625 Obtiene una copia del First Header
627
626
628 Affected:
627 Affected:
629 self.systemHeaderObj
628 self.systemHeaderObj
630 self.radarControllerHeaderObj
629 self.radarControllerHeaderObj
631 self.dtype
630 self.dtype
632
631
633 Return:
632 Return:
634 None
633 None
635 """
634 """
636
635
637 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
636 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
638 self.systemHeaderObj.nChannels = self.dataOut.nChannels
637 self.systemHeaderObj.nChannels = self.dataOut.nChannels
639 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
638 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
640
639
641 self.processingHeaderObj.dtype = 1 # Spectra
640 self.processingHeaderObj.dtype = 1 # Spectra
642 self.processingHeaderObj.blockSize = self.__getBlockSize()
641 self.processingHeaderObj.blockSize = self.__getBlockSize()
643 self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
642 self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
644 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
643 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
645 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
644 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
646 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
645 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
647 self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
646 self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
648 self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
647 self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
649 self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
648 self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
650
649
651 if self.processingHeaderObj.totalSpectra > 0:
650 if self.processingHeaderObj.totalSpectra > 0:
652 channelList = []
651 channelList = []
653 for channel in range(self.dataOut.nChannels):
652 for channel in range(self.dataOut.nChannels):
654 channelList.append(channel)
653 channelList.append(channel)
655 channelList.append(channel)
654 channelList.append(channel)
656
655
657 pairsList = []
656 pairsList = []
658 if self.dataOut.nPairs > 0:
657 if self.dataOut.nPairs > 0:
659 for pair in self.dataOut.pairsList:
658 for pair in self.dataOut.pairsList:
660 pairsList.append(pair[0])
659 pairsList.append(pair[0])
661 pairsList.append(pair[1])
660 pairsList.append(pair[1])
662
661
663 spectraComb = channelList + pairsList
662 spectraComb = channelList + pairsList
664 spectraComb = numpy.array(spectraComb, dtype="u1")
663 spectraComb = numpy.array(spectraComb, dtype="u1")
665 self.processingHeaderObj.spectraComb = spectraComb
664 self.processingHeaderObj.spectraComb = spectraComb
666
665
667 if self.dataOut.code is not None:
666 if self.dataOut.code is not None:
668 self.processingHeaderObj.code = self.dataOut.code
667 self.processingHeaderObj.code = self.dataOut.code
669 self.processingHeaderObj.nCode = self.dataOut.nCode
668 self.processingHeaderObj.nCode = self.dataOut.nCode
670 self.processingHeaderObj.nBaud = self.dataOut.nBaud
669 self.processingHeaderObj.nBaud = self.dataOut.nBaud
671
670
672 if self.processingHeaderObj.nWindows != 0:
671 if self.processingHeaderObj.nWindows != 0:
673 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
672 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
674 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
673 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
675 self.processingHeaderObj.nHeights = self.dataOut.nHeights
674 self.processingHeaderObj.nHeights = self.dataOut.nHeights
676 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
675 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
677
676
678 self.processingHeaderObj.processFlags = self.getProcessFlags()
677 self.processingHeaderObj.processFlags = self.getProcessFlags()
679
678
680 self.setBasicHeader() No newline at end of file
679 self.setBasicHeader()
Show file before | Show file after | Hide comments
@@ -1,765 +1,765
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6
6
7 import numpy
7 import numpy
8
8
9 from .jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
9 from .jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 from schainpy.model.data.jrodata import Voltage
12 from schainpy.model.data.jrodata import Voltage
13 import zmq
13 import zmq
14 import tempfile
14 import tempfile
15 from io import StringIO
15 from io import StringIO
16 # from _sha import blocksize
16 # from _sha import blocksize
17
17
18 @MPDecorator
18 @MPDecorator
19 class VoltageReader(JRODataReader, ProcessingUnit):
19 class VoltageReader(JRODataReader, ProcessingUnit):
20 """
20 """
21 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
21 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
22 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
22 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
23 perfiles*alturas*canales) son almacenados en la variable "buffer".
23 perfiles*alturas*canales) son almacenados en la variable "buffer".
24
24
25 perfiles * alturas * canales
25 perfiles * alturas * canales
26
26
27 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
27 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
28 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
28 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
29 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
29 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
30 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
30 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
31
31
32 Example:
32 Example:
33
33
34 dpath = "/home/myuser/data"
34 dpath = "/home/myuser/data"
35
35
36 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
36 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
37
37
38 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
38 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
39
39
40 readerObj = VoltageReader()
40 readerObj = VoltageReader()
41
41
42 readerObj.setup(dpath, startTime, endTime)
42 readerObj.setup(dpath, startTime, endTime)
43
43
44 while(True):
44 while(True):
45
45
46 #to get one profile
46 #to get one profile
47 profile = readerObj.getData()
47 profile = readerObj.getData()
48
48
49 #print the profile
49 #print the profile
50 print profile
50 print profile
51
51
52 #If you want to see all datablock
52 #If you want to see all datablock
53 print readerObj.datablock
53 print readerObj.datablock
54
54
55 if readerObj.flagNoMoreFiles:
55 if readerObj.flagNoMoreFiles:
56 break
56 break
57
57
58 """
58 """
59
59
60 ext = ".r"
60 ext = ".r"
61
61
62 optchar = "D"
62 optchar = "D"
63 dataOut = None
63 dataOut = None
64
64
65 def __init__(self):#, **kwargs):
65 def __init__(self):#, **kwargs):
66 """
66 """
67 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
67 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
68
68
69 Input:
69 Input:
70 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
70 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
71 almacenar un perfil de datos cada vez que se haga un requerimiento
71 almacenar un perfil de datos cada vez que se haga un requerimiento
72 (getData). El perfil sera obtenido a partir del buffer de datos,
72 (getData). El perfil sera obtenido a partir del buffer de datos,
73 si el buffer esta vacio se hara un nuevo proceso de lectura de un
73 si el buffer esta vacio se hara un nuevo proceso de lectura de un
74 bloque de datos.
74 bloque de datos.
75 Si este parametro no es pasado se creara uno internamente.
75 Si este parametro no es pasado se creara uno internamente.
76
76
77 Variables afectadas:
77 Variables afectadas:
78 self.dataOut
78 self.dataOut
79
79
80 Return:
80 Return:
81 None
81 None
82 """
82 """
83
83
84 ProcessingUnit.__init__(self)#, **kwargs)
84 ProcessingUnit.__init__(self)#, **kwargs)
85
85
86 self.isConfig = False
86 self.isConfig = False
87
87
88 self.datablock = None
88 self.datablock = None
89
89
90 self.utc = 0
90 self.utc = 0
91
91
92 self.ext = ".r"
92 self.ext = ".r"
93
93
94 self.optchar = "D"
94 self.optchar = "D"
95
95
96 self.basicHeaderObj = BasicHeader(LOCALTIME)
96 self.basicHeaderObj = BasicHeader(LOCALTIME)
97
97
98 self.systemHeaderObj = SystemHeader()
98 self.systemHeaderObj = SystemHeader()
99
99
100 self.radarControllerHeaderObj = RadarControllerHeader()
100 self.radarControllerHeaderObj = RadarControllerHeader()
101
101
102 self.processingHeaderObj = ProcessingHeader()
102 self.processingHeaderObj = ProcessingHeader()
103
103
104 self.online = 0
104 self.online = 0
105
105
106 self.fp = None
106 self.fp = None
107
107
108 self.idFile = None
108 self.idFile = None
109
109
110 self.dtype = None
110 self.dtype = None
111
111
112 self.fileSizeByHeader = None
112 self.fileSizeByHeader = None
113
113
114 self.filenameList = []
114 self.filenameList = []
115
115
116 self.filename = None
116 self.filename = None
117
117
118 self.fileSize = None
118 self.fileSize = None
119
119
120 self.firstHeaderSize = 0
120 self.firstHeaderSize = 0
121
121
122 self.basicHeaderSize = 24
122 self.basicHeaderSize = 24
123
123
124 self.pathList = []
124 self.pathList = []
125
125
126 self.filenameList = []
126 self.filenameList = []
127
127
128 self.lastUTTime = 0
128 self.lastUTTime = 0
129
129
130 self.maxTimeStep = 30
130 self.maxTimeStep = 30
131
131
132 self.flagNoMoreFiles = 0
132 self.flagNoMoreFiles = 0
133
133
134 self.set = 0
134 self.set = 0
135
135
136 self.path = None
136 self.path = None
137
137
138 self.profileIndex = 2**32 - 1
138 self.profileIndex = 2**32 - 1
139
139
140 self.delay = 3 # seconds
140 self.delay = 3 # seconds
141
141
142 self.nTries = 3 # quantity tries
142 self.nTries = 3 # quantity tries
143
143
144 self.nFiles = 3 # number of files for searching
144 self.nFiles = 3 # number of files for searching
145
145
146 self.nReadBlocks = 0
146 self.nReadBlocks = 0
147
147
148 self.flagIsNewFile = 1
148 self.flagIsNewFile = 1
149
149
150 self.__isFirstTimeOnline = 1
150 self.__isFirstTimeOnline = 1
151
151
152 # self.ippSeconds = 0
152 # self.ippSeconds = 0
153
153
154 self.flagDiscontinuousBlock = 0
154 self.flagDiscontinuousBlock = 0
155
155
156 self.flagIsNewBlock = 0
156 self.flagIsNewBlock = 0
157
157
158 self.nTotalBlocks = 0
158 self.nTotalBlocks = 0
159
159
160 self.blocksize = 0
160 self.blocksize = 0
161
161
162 self.dataOut = self.createObjByDefault()
162 self.dataOut = self.createObjByDefault()
163
163
164 self.nTxs = 1
164 self.nTxs = 1
165
165
166 self.txIndex = 0
166 self.txIndex = 0
167
167
168 def createObjByDefault(self):
168 def createObjByDefault(self):
169
169
170 dataObj = Voltage()
170 dataObj = Voltage()
171
171
172 return dataObj
172 return dataObj
173
173
174 def __hasNotDataInBuffer(self):
174 def __hasNotDataInBuffer(self):
175
175
176 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock * self.nTxs:
176 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock * self.nTxs:
177 return 1
177 return 1
178
178
179 return 0
179 return 0
180
180
181 def getBlockDimension(self):
181 def getBlockDimension(self):
182 """
182 """
183 Obtiene la cantidad de puntos a leer por cada bloque de datos
183 Obtiene la cantidad de puntos a leer por cada bloque de datos
184
184
185 Affected:
185 Affected:
186 self.blocksize
186 self.blocksize
187
187
188 Return:
188 Return:
189 None
189 None
190 """
190 """
191 pts2read = self.processingHeaderObj.profilesPerBlock * \
191 pts2read = self.processingHeaderObj.profilesPerBlock * \
192 self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
192 self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
193 self.blocksize = pts2read
193 self.blocksize = pts2read
194
194
195 def readBlock(self):
195 def readBlock(self):
196 """
196 """
197 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
197 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
198 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
198 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
199 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
199 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
200 es seteado a 0
200 es seteado a 0
201
201
202 Inputs:
202 Inputs:
203 None
203 None
204
204
205 Return:
205 Return:
206 None
206 None
207
207
208 Affected:
208 Affected:
209 self.profileIndex
209 self.profileIndex
210 self.datablock
210 self.datablock
211 self.flagIsNewFile
211 self.flagIsNewFile
212 self.flagIsNewBlock
212 self.flagIsNewBlock
213 self.nTotalBlocks
213 self.nTotalBlocks
214
214
215 Exceptions:
215 Exceptions:
216 Si un bloque leido no es un bloque valido
216 Si un bloque leido no es un bloque valido
217 """
217 """
218
218
219 # if self.server is not None:
219 # if self.server is not None:
220 # self.zBlock = self.receiver.recv()
220 # self.zBlock = self.receiver.recv()
221 # self.zHeader = self.zBlock[:24]
221 # self.zHeader = self.zBlock[:24]
222 # self.zDataBlock = self.zBlock[24:]
222 # self.zDataBlock = self.zBlock[24:]
223 # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
223 # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
224 # self.processingHeaderObj.profilesPerBlock = 240
224 # self.processingHeaderObj.profilesPerBlock = 240
225 # self.processingHeaderObj.nHeights = 248
225 # self.processingHeaderObj.nHeights = 248
226 # self.systemHeaderObj.nChannels
226 # self.systemHeaderObj.nChannels
227 # else:
227 # else:
228 current_pointer_location = self.fp.tell()
228 current_pointer_location = self.fp.tell()
229 junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
229 junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
230
230
231 try:
231 try:
232 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
232 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
233 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
233 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
234 except:
234 except:
235 # print "The read block (%3d) has not enough data" %self.nReadBlocks
235 # print "The read block (%3d) has not enough data" %self.nReadBlocks
236
236
237 if self.waitDataBlock(pointer_location=current_pointer_location):
237 if self.waitDataBlock(pointer_location=current_pointer_location):
238 junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
238 junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
239 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
239 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
240 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
240 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
241 # return 0
241 # return 0
242
242
243 # Dimensions : nChannels, nProfiles, nSamples
243 # Dimensions : nChannels, nProfiles, nSamples
244
244
245 junk = numpy.transpose(junk, (2, 0, 1))
245 junk = numpy.transpose(junk, (2, 0, 1))
246 self.datablock = junk['real'] + junk['imag'] * 1j
246 self.datablock = junk['real'] + junk['imag'] * 1j
247
247
248 self.profileIndex = 0
248 self.profileIndex = 0
249
249
250 self.flagIsNewFile = 0
250 self.flagIsNewFile = 0
251 self.flagIsNewBlock = 1
251 self.flagIsNewBlock = 1
252
252
253 self.nTotalBlocks += 1
253 self.nTotalBlocks += 1
254 self.nReadBlocks += 1
254 self.nReadBlocks += 1
255
255
256 return 1
256 return 1
257
257
258 def getFirstHeader(self):
258 def getFirstHeader(self):
259
259
260 self.getBasicHeader()
260 self.getBasicHeader()
261
261
262 self.dataOut.processingHeaderObj = self.processingHeaderObj.copy()
262 self.dataOut.processingHeaderObj = self.processingHeaderObj.copy()
263
263
264 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
264 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
265
265
266 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
266 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
267
267
268 if self.nTxs > 1:
268 if self.nTxs > 1:
269 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
269 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
270 # Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
270 # Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
271
271
272 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
272 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
273 #
273 #
274 # if self.radarControllerHeaderObj.code is not None:
274 # if self.radarControllerHeaderObj.code is not None:
275 #
275 #
276 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
276 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
277 #
277 #
278 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
278 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
279 #
279 #
280 # self.dataOut.code = self.radarControllerHeaderObj.code
280 # self.dataOut.code = self.radarControllerHeaderObj.code
281
281
282 self.dataOut.dtype = self.dtype
282 self.dataOut.dtype = self.dtype
283
283
284 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
284 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
285
285
286 self.dataOut.heightList = numpy.arange(
286 self.dataOut.heightList = numpy.arange(
287 self.processingHeaderObj.nHeights) * self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
287 self.processingHeaderObj.nHeights) * self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
288
288
289 self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
289 self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
290
290
291 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
291 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
292
292
293 # asumo q la data no esta decodificada
293 # asumo q la data no esta decodificada
294 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode
294 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode
295
295
296 # asumo q la data no esta sin flip
296 # asumo q la data no esta sin flip
297 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip
297 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip
298
298
299 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
299 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
300
300
301 def reshapeData(self):
301 def reshapeData(self):
302
302
303 if self.nTxs < 0:
303 if self.nTxs < 0:
304 return
304 return
305
305
306 if self.nTxs == 1:
306 if self.nTxs == 1:
307 return
307 return
308
308
309 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1. / self.nTxs) != 0:
309 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1. / self.nTxs) != 0:
310 raise ValueError("1./nTxs (=%f), should be a multiple of nProfiles (=%d)" % (
310 raise ValueError("1./nTxs (=%f), should be a multiple of nProfiles (=%d)" % (
311 1. / self.nTxs, self.processingHeaderObj.profilesPerBlock))
311 1. / self.nTxs, self.processingHeaderObj.profilesPerBlock))
312
312
313 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
313 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
314 raise ValueError("nTxs (=%d), should be a multiple of nHeights (=%d)" % (
314 raise ValueError("nTxs (=%d), should be a multiple of nHeights (=%d)" % (
315 self.nTxs, self.processingHeaderObj.nHeights))
315 self.nTxs, self.processingHeaderObj.nHeights))
316
316
317 self.datablock = self.datablock.reshape(
317 self.datablock = self.datablock.reshape(
318 (self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock * self.nTxs, self.processingHeaderObj.nHeights / self.nTxs))
318 (self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock * self.nTxs, self.processingHeaderObj.nHeights / self.nTxs))
319
319
320 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock * self.nTxs
320 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock * self.nTxs
321 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights / self.nTxs) * \
321 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights / self.nTxs) * \
322 self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
322 self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
323 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
323 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
324
324
325 return
325 return
326
326
327 def readFirstHeaderFromServer(self):
327 def readFirstHeaderFromServer(self):
328
328
329 self.getFirstHeader()
329 self.getFirstHeader()
330
330
331 self.firstHeaderSize = self.basicHeaderObj.size
331 self.firstHeaderSize = self.basicHeaderObj.size
332
332
333 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
333 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
334 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
334 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
335 if datatype == 0:
335 if datatype == 0:
336 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
336 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
337 elif datatype == 1:
337 elif datatype == 1:
338 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
338 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
339 elif datatype == 2:
339 elif datatype == 2:
340 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
340 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
341 elif datatype == 3:
341 elif datatype == 3:
342 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
342 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
343 elif datatype == 4:
343 elif datatype == 4:
344 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
344 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
345 elif datatype == 5:
345 elif datatype == 5:
346 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
346 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
347 else:
347 else:
348 raise ValueError('Data type was not defined')
348 raise ValueError('Data type was not defined')
349
349
350 self.dtype = datatype_str
350 self.dtype = datatype_str
351 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
351 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
352 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
352 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
353 self.firstHeaderSize + self.basicHeaderSize * \
353 self.firstHeaderSize + self.basicHeaderSize * \
354 (self.processingHeaderObj.dataBlocksPerFile - 1)
354 (self.processingHeaderObj.dataBlocksPerFile - 1)
355 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
355 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
356 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
356 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
357 self.getBlockDimension()
357 self.getBlockDimension()
358
358
359 def getFromServer(self):
359 def getFromServer(self):
360 self.flagDiscontinuousBlock = 0
360 self.flagDiscontinuousBlock = 0
361 self.profileIndex = 0
361 self.profileIndex = 0
362 self.flagIsNewBlock = 1
362 self.flagIsNewBlock = 1
363 self.dataOut.flagNoData = False
363 self.dataOut.flagNoData = False
364 self.nTotalBlocks += 1
364 self.nTotalBlocks += 1
365 self.nReadBlocks += 1
365 self.nReadBlocks += 1
366 self.blockPointer = 0
366 self.blockPointer = 0
367
367
368 block = self.receiver.recv()
368 block = self.receiver.recv()
369
369
370 self.basicHeaderObj.read(block[self.blockPointer:])
370 self.basicHeaderObj.read(block[self.blockPointer:])
371 self.blockPointer += self.basicHeaderObj.length
371 self.blockPointer += self.basicHeaderObj.length
372 self.systemHeaderObj.read(block[self.blockPointer:])
372 self.systemHeaderObj.read(block[self.blockPointer:])
373 self.blockPointer += self.systemHeaderObj.length
373 self.blockPointer += self.systemHeaderObj.length
374 self.radarControllerHeaderObj.read(block[self.blockPointer:])
374 self.radarControllerHeaderObj.read(block[self.blockPointer:])
375 self.blockPointer += self.radarControllerHeaderObj.length
375 self.blockPointer += self.radarControllerHeaderObj.length
376 self.processingHeaderObj.read(block[self.blockPointer:])
376 self.processingHeaderObj.read(block[self.blockPointer:])
377 self.blockPointer += self.processingHeaderObj.length
377 self.blockPointer += self.processingHeaderObj.length
378 self.readFirstHeaderFromServer()
378 self.readFirstHeaderFromServer()
379
379
380 timestamp = self.basicHeaderObj.get_datatime()
380 timestamp = self.basicHeaderObj.get_datatime()
381 print('[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp))
381 print('[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp))
382 current_pointer_location = self.blockPointer
382 current_pointer_location = self.blockPointer
383 junk = numpy.fromstring(
383 junk = numpy.fromstring(
384 block[self.blockPointer:], self.dtype, self.blocksize)
384 block[self.blockPointer:], self.dtype, self.blocksize)
385
385
386 try:
386 try:
387 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
387 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
388 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
388 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
389 except:
389 except:
390 # print "The read block (%3d) has not enough data" %self.nReadBlocks
390 # print "The read block (%3d) has not enough data" %self.nReadBlocks
391 if self.waitDataBlock(pointer_location=current_pointer_location):
391 if self.waitDataBlock(pointer_location=current_pointer_location):
392 junk = numpy.fromstring(
392 junk = numpy.fromstring(
393 block[self.blockPointer:], self.dtype, self.blocksize)
393 block[self.blockPointer:], self.dtype, self.blocksize)
394 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
394 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
395 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
395 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
396 # return 0
396 # return 0
397
397
398 # Dimensions : nChannels, nProfiles, nSamples
398 # Dimensions : nChannels, nProfiles, nSamples
399
399
400 junk = numpy.transpose(junk, (2, 0, 1))
400 junk = numpy.transpose(junk, (2, 0, 1))
401 self.datablock = junk['real'] + junk['imag'] * 1j
401 self.datablock = junk['real'] + junk['imag'] * 1j
402 self.profileIndex = 0
402 self.profileIndex = 0
403 if self.selBlocksize == None:
403 if self.selBlocksize == None:
404 self.selBlocksize = self.dataOut.nProfiles
404 self.selBlocksize = self.dataOut.nProfiles
405 if self.selBlocktime != None:
405 if self.selBlocktime != None:
406 if self.dataOut.nCohInt is not None:
406 if self.dataOut.nCohInt is not None:
407 nCohInt = self.dataOut.nCohInt
407 nCohInt = self.dataOut.nCohInt
408 else:
408 else:
409 nCohInt = 1
409 nCohInt = 1
410 self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
410 self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
411 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
411 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
412 self.dataOut.data = self.datablock[:,
412 self.dataOut.data = self.datablock[:,
413 self.profileIndex:self.profileIndex + self.selBlocksize, :]
413 self.profileIndex:self.profileIndex + self.selBlocksize, :]
414 datasize = self.dataOut.data.shape[1]
414 datasize = self.dataOut.data.shape[1]
415 if datasize < self.selBlocksize:
415 if datasize < self.selBlocksize:
416 buffer = numpy.zeros(
416 buffer = numpy.zeros(
417 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
417 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
418 buffer[:, :datasize, :] = self.dataOut.data
418 buffer[:, :datasize, :] = self.dataOut.data
419 self.dataOut.data = buffer
419 self.dataOut.data = buffer
420 self.profileIndex = blockIndex
420 self.profileIndex = blockIndex
421
421
422 self.dataOut.flagDataAsBlock = True
422 self.dataOut.flagDataAsBlock = True
423 self.flagIsNewBlock = 1
423 self.flagIsNewBlock = 1
424 self.dataOut.realtime = self.online
424 self.dataOut.realtime = self.online
425
425
426 return self.dataOut.data
426 return self.dataOut.data
427
427
428 def getData(self):
428 def getData(self):
429 """
429 """
430 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
430 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
431 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
431 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
432 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
432 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
433 "readNextBlock"
433 "readNextBlock"
434
434
435 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
435 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
436
436
437 Return:
437 Return:
438
438
439 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
439 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
440 es igual al total de perfiles leidos desde el archivo.
440 es igual al total de perfiles leidos desde el archivo.
441
441
442 Si self.getByBlock == False:
442 Si self.getByBlock == False:
443
443
444 self.dataOut.data = buffer[:, thisProfile, :]
444 self.dataOut.data = buffer[:, thisProfile, :]
445
445
446 shape = [nChannels, nHeis]
446 shape = [nChannels, nHeis]
447
447
448 Si self.getByBlock == True:
448 Si self.getByBlock == True:
449
449
450 self.dataOut.data = buffer[:, :, :]
450 self.dataOut.data = buffer[:, :, :]
451
451
452 shape = [nChannels, nProfiles, nHeis]
452 shape = [nChannels, nProfiles, nHeis]
453
453
454 Variables afectadas:
454 Variables afectadas:
455 self.dataOut
455 self.dataOut
456 self.profileIndex
456 self.profileIndex
457
457
458 Affected:
458 Affected:
459 self.dataOut
459 self.dataOut
460 self.profileIndex
460 self.profileIndex
461 self.flagDiscontinuousBlock
461 self.flagDiscontinuousBlock
462 self.flagIsNewBlock
462 self.flagIsNewBlock
463 """
463 """
464 if self.flagNoMoreFiles:
464 if self.flagNoMoreFiles:
465 self.dataOut.flagNoData = True
465 self.dataOut.flagNoData = True
466 print('Process finished')
466 print('Process finished')
467 return 0
467 return 0
468 self.flagDiscontinuousBlock = 0
468 self.flagDiscontinuousBlock = 0
469 self.flagIsNewBlock = 0
469 self.flagIsNewBlock = 0
470 if self.__hasNotDataInBuffer():
470 if self.__hasNotDataInBuffer():
471 if not(self.readNextBlock()):
471 if not(self.readNextBlock()):
472 return 0
472 return 0
473
473
474 self.getFirstHeader()
474 self.getFirstHeader()
475
475
476 self.reshapeData()
476 self.reshapeData()
477 if self.datablock is None:
477 if self.datablock is None:
478 self.dataOut.flagNoData = True
478 self.dataOut.flagNoData = True
479 return 0
479 return 0
480
480
481 if not self.getByBlock:
481 if not self.getByBlock:
482
482
483 """
483 """
484 Return profile by profile
484 Return profile by profile
485
485
486 If nTxs > 1 then one profile is divided by nTxs and number of total
486 If nTxs > 1 then one profile is divided by nTxs and number of total
487 blocks is increased by nTxs (nProfiles *= nTxs)
487 blocks is increased by nTxs (nProfiles *= nTxs)
488 """
488 """
489 self.dataOut.flagDataAsBlock = False
489 self.dataOut.flagDataAsBlock = False
490 self.dataOut.data = self.datablock[:, self.profileIndex, :]
490 self.dataOut.data = self.datablock[:, self.profileIndex, :]
491 self.dataOut.profileIndex = self.profileIndex
491 self.dataOut.profileIndex = self.profileIndex
492
492
493 self.profileIndex += 1
493 self.profileIndex += 1
494
494
495 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
495 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
496 # """
496 # """
497 # Return all block
497 # Return all block
498 # """
498 # """
499 # self.dataOut.flagDataAsBlock = True
499 # self.dataOut.flagDataAsBlock = True
500 # self.dataOut.data = self.datablock
500 # self.dataOut.data = self.datablock
501 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
501 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
502 #
502 #
503 # self.profileIndex = self.dataOut.nProfiles
503 # self.profileIndex = self.dataOut.nProfiles
504
504
505 else:
505 else:
506 """
506 """
507 Return a block
507 Return a block
508 """
508 """
509 if self.selBlocksize == None:
509 if self.selBlocksize == None:
510 self.selBlocksize = self.dataOut.nProfiles
510 self.selBlocksize = self.dataOut.nProfiles
511 if self.selBlocktime != None:
511 if self.selBlocktime != None:
512 if self.dataOut.nCohInt is not None:
512 if self.dataOut.nCohInt is not None:
513 nCohInt = self.dataOut.nCohInt
513 nCohInt = self.dataOut.nCohInt
514 else:
514 else:
515 nCohInt = 1
515 nCohInt = 1
516 self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
516 self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
517 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
517 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
518
518
519 self.dataOut.data = self.datablock[:,
519 self.dataOut.data = self.datablock[:,
520 self.profileIndex:self.profileIndex + self.selBlocksize, :]
520 self.profileIndex:self.profileIndex + self.selBlocksize, :]
521 self.profileIndex += self.selBlocksize
521 self.profileIndex += self.selBlocksize
522 datasize = self.dataOut.data.shape[1]
522 datasize = self.dataOut.data.shape[1]
523
523
524 if datasize < self.selBlocksize:
524 if datasize < self.selBlocksize:
525 buffer = numpy.zeros(
525 buffer = numpy.zeros(
526 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
526 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
527 buffer[:, :datasize, :] = self.dataOut.data
527 buffer[:, :datasize, :] = self.dataOut.data
528
528
529 while datasize < self.selBlocksize: # Not enough profiles to fill the block
529 while datasize < self.selBlocksize: # Not enough profiles to fill the block
530 if not(self.readNextBlock()):
530 if not(self.readNextBlock()):
531 return 0
531 return 0
532 self.getFirstHeader()
532 self.getFirstHeader()
533 self.reshapeData()
533 self.reshapeData()
534 if self.datablock is None:
534 if self.datablock is None:
535 self.dataOut.flagNoData = True
535 self.dataOut.flagNoData = True
536 return 0
536 return 0
537 # stack data
537 # stack data
538 blockIndex = self.selBlocksize - datasize
538 blockIndex = self.selBlocksize - datasize
539 datablock1 = self.datablock[:, :blockIndex, :]
539 datablock1 = self.datablock[:, :blockIndex, :]
540
540
541 buffer[:, datasize:datasize +
541 buffer[:, datasize:datasize +
542 datablock1.shape[1], :] = datablock1
542 datablock1.shape[1], :] = datablock1
543 datasize += datablock1.shape[1]
543 datasize += datablock1.shape[1]
544
544
545 self.dataOut.data = buffer
545 self.dataOut.data = buffer
546 self.profileIndex = blockIndex
546 self.profileIndex = blockIndex
547
547
548 self.dataOut.flagDataAsBlock = True
548 self.dataOut.flagDataAsBlock = True
549 self.dataOut.nProfiles = self.dataOut.data.shape[1]
549 self.dataOut.nProfiles = self.dataOut.data.shape[1]
550
550
551 self.dataOut.flagNoData = False
551 self.dataOut.flagNoData = False
552
552
553 self.getBasicHeader()
553 self.getBasicHeader()
554
554
555 self.dataOut.realtime = self.online
555 self.dataOut.realtime = self.online
556
556
557 return self.dataOut.data
557 return self.dataOut.data
558
558
559
559 @MPDecorator
560 class VoltageWriter(JRODataWriter, Operation):
560 class VoltageWriter(JRODataWriter, Operation):
561 """
561 """
562 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
562 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
563 de los datos siempre se realiza por bloques.
563 de los datos siempre se realiza por bloques.
564 """
564 """
565
565
566 ext = ".r"
566 ext = ".r"
567
567
568 optchar = "D"
568 optchar = "D"
569
569
570 shapeBuffer = None
570 shapeBuffer = None
571
571
572 def __init__(self, **kwargs):
572 def __init__(self):#, **kwargs):
573 """
573 """
574 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
574 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
575
575
576 Affected:
576 Affected:
577 self.dataOut
577 self.dataOut
578
578
579 Return: None
579 Return: None
580 """
580 """
581 Operation.__init__(self, **kwargs)
581 Operation.__init__(self)#, **kwargs)
582
582
583 self.nTotalBlocks = 0
583 self.nTotalBlocks = 0
584
584
585 self.profileIndex = 0
585 self.profileIndex = 0
586
586
587 self.isConfig = False
587 self.isConfig = False
588
588
589 self.fp = None
589 self.fp = None
590
590
591 self.flagIsNewFile = 1
591 self.flagIsNewFile = 1
592
592
593 self.blockIndex = 0
593 self.blockIndex = 0
594
594
595 self.flagIsNewBlock = 0
595 self.flagIsNewBlock = 0
596
596
597 self.setFile = None
597 self.setFile = None
598
598
599 self.dtype = None
599 self.dtype = None
600
600
601 self.path = None
601 self.path = None
602
602
603 self.filename = None
603 self.filename = None
604
604
605 self.basicHeaderObj = BasicHeader(LOCALTIME)
605 self.basicHeaderObj = BasicHeader(LOCALTIME)
606
606
607 self.systemHeaderObj = SystemHeader()
607 self.systemHeaderObj = SystemHeader()
608
608
609 self.radarControllerHeaderObj = RadarControllerHeader()
609 self.radarControllerHeaderObj = RadarControllerHeader()
610
610
611 self.processingHeaderObj = ProcessingHeader()
611 self.processingHeaderObj = ProcessingHeader()
612
612
613 def hasAllDataInBuffer(self):
613 def hasAllDataInBuffer(self):
614 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
614 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
615 return 1
615 return 1
616 return 0
616 return 0
617
617
618 def setBlockDimension(self):
618 def setBlockDimension(self):
619 """
619 """
620 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
620 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
621
621
622 Affected:
622 Affected:
623 self.shape_spc_Buffer
623 self.shape_spc_Buffer
624 self.shape_cspc_Buffer
624 self.shape_cspc_Buffer
625 self.shape_dc_Buffer
625 self.shape_dc_Buffer
626
626
627 Return: None
627 Return: None
628 """
628 """
629 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
629 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
630 self.processingHeaderObj.nHeights,
630 self.processingHeaderObj.nHeights,
631 self.systemHeaderObj.nChannels)
631 self.systemHeaderObj.nChannels)
632
632
633 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
633 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
634 self.processingHeaderObj.profilesPerBlock,
634 self.processingHeaderObj.profilesPerBlock,
635 self.processingHeaderObj.nHeights),
635 self.processingHeaderObj.nHeights),
636 dtype=numpy.dtype('complex64'))
636 dtype=numpy.dtype('complex64'))
637
637
638 def writeBlock(self):
638 def writeBlock(self):
639 """
639 """
640 Escribe el buffer en el file designado
640 Escribe el buffer en el file designado
641
641
642 Affected:
642 Affected:
643 self.profileIndex
643 self.profileIndex
644 self.flagIsNewFile
644 self.flagIsNewFile
645 self.flagIsNewBlock
645 self.flagIsNewBlock
646 self.nTotalBlocks
646 self.nTotalBlocks
647 self.blockIndex
647 self.blockIndex
648
648
649 Return: None
649 Return: None
650 """
650 """
651 data = numpy.zeros(self.shapeBuffer, self.dtype)
651 data = numpy.zeros(self.shapeBuffer, self.dtype)
652
652
653 junk = numpy.transpose(self.datablock, (1, 2, 0))
653 junk = numpy.transpose(self.datablock, (1, 2, 0))
654
654
655 data['real'] = junk.real
655 data['real'] = junk.real
656 data['imag'] = junk.imag
656 data['imag'] = junk.imag
657
657
658 data = data.reshape((-1))
658 data = data.reshape((-1))
659
659
660 data.tofile(self.fp)
660 data.tofile(self.fp)
661
661
662 self.datablock.fill(0)
662 self.datablock.fill(0)
663
663
664 self.profileIndex = 0
664 self.profileIndex = 0
665 self.flagIsNewFile = 0
665 self.flagIsNewFile = 0
666 self.flagIsNewBlock = 1
666 self.flagIsNewBlock = 1
667
667
668 self.blockIndex += 1
668 self.blockIndex += 1
669 self.nTotalBlocks += 1
669 self.nTotalBlocks += 1
670
670
671 # print "[Writing] Block = %04d" %self.blockIndex
671 # print "[Writing] Block = %04d" %self.blockIndex
672
672
673 def putData(self):
673 def putData(self):
674 """
674 """
675 Setea un bloque de datos y luego los escribe en un file
675 Setea un bloque de datos y luego los escribe en un file
676
676
677 Affected:
677 Affected:
678 self.flagIsNewBlock
678 self.flagIsNewBlock
679 self.profileIndex
679 self.profileIndex
680
680
681 Return:
681 Return:
682 0 : Si no hay data o no hay mas files que puedan escribirse
682 0 : Si no hay data o no hay mas files que puedan escribirse
683 1 : Si se escribio la data de un bloque en un file
683 1 : Si se escribio la data de un bloque en un file
684 """
684 """
685 if self.dataOut.flagNoData:
685 if self.dataOut.flagNoData:
686 return 0
686 return 0
687
687
688 self.flagIsNewBlock = 0
688 self.flagIsNewBlock = 0
689
689
690 if self.dataOut.flagDiscontinuousBlock:
690 if self.dataOut.flagDiscontinuousBlock:
691 self.datablock.fill(0)
691 self.datablock.fill(0)
692 self.profileIndex = 0
692 self.profileIndex = 0
693 self.setNextFile()
693 self.setNextFile()
694
694
695 if self.profileIndex == 0:
695 if self.profileIndex == 0:
696 self.setBasicHeader()
696 self.setBasicHeader()
697
697
698 self.datablock[:, self.profileIndex, :] = self.dataOut.data
698 self.datablock[:, self.profileIndex, :] = self.dataOut.data
699
699
700 self.profileIndex += 1
700 self.profileIndex += 1
701
701
702 if self.hasAllDataInBuffer():
702 if self.hasAllDataInBuffer():
703 # if self.flagIsNewFile:
703 # if self.flagIsNewFile:
704 self.writeNextBlock()
704 self.writeNextBlock()
705 # self.setFirstHeader()
705 # self.setFirstHeader()
706
706
707 return 1
707 return 1
708
708
709 def __getBlockSize(self):
709 def __getBlockSize(self):
710 '''
710 '''
711 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
711 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
712 '''
712 '''
713
713
714 dtype_width = self.getDtypeWidth()
714 dtype_width = self.getDtypeWidth()
715
715
716 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels *
716 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels *
717 self.profilesPerBlock * dtype_width * 2)
717 self.profilesPerBlock * dtype_width * 2)
718
718
719 return blocksize
719 return blocksize
720
720
721 def setFirstHeader(self):
721 def setFirstHeader(self):
722 """
722 """
723 Obtiene una copia del First Header
723 Obtiene una copia del First Header
724
724
725 Affected:
725 Affected:
726 self.systemHeaderObj
726 self.systemHeaderObj
727 self.radarControllerHeaderObj
727 self.radarControllerHeaderObj
728 self.dtype
728 self.dtype
729
729
730 Return:
730 Return:
731 None
731 None
732 """
732 """
733
733
734 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
734 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
735 self.systemHeaderObj.nChannels = self.dataOut.nChannels
735 self.systemHeaderObj.nChannels = self.dataOut.nChannels
736 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
736 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
737
737
738 self.processingHeaderObj.dtype = 0 # Voltage
738 self.processingHeaderObj.dtype = 0 # Voltage
739 self.processingHeaderObj.blockSize = self.__getBlockSize()
739 self.processingHeaderObj.blockSize = self.__getBlockSize()
740 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
740 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
741 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
741 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
742 # podria ser 1 o self.dataOut.processingHeaderObj.nWindows
742 # podria ser 1 o self.dataOut.processingHeaderObj.nWindows
743 self.processingHeaderObj.nWindows = 1
743 self.processingHeaderObj.nWindows = 1
744 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
744 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
745 # Cuando la data de origen es de tipo Voltage
745 # Cuando la data de origen es de tipo Voltage
746 self.processingHeaderObj.nIncohInt = 1
746 self.processingHeaderObj.nIncohInt = 1
747 # Cuando la data de origen es de tipo Voltage
747 # Cuando la data de origen es de tipo Voltage
748 self.processingHeaderObj.totalSpectra = 0
748 self.processingHeaderObj.totalSpectra = 0
749
749
750 if self.dataOut.code is not None:
750 if self.dataOut.code is not None:
751 self.processingHeaderObj.code = self.dataOut.code
751 self.processingHeaderObj.code = self.dataOut.code
752 self.processingHeaderObj.nCode = self.dataOut.nCode
752 self.processingHeaderObj.nCode = self.dataOut.nCode
753 self.processingHeaderObj.nBaud = self.dataOut.nBaud
753 self.processingHeaderObj.nBaud = self.dataOut.nBaud
754
754
755 if self.processingHeaderObj.nWindows != 0:
755 if self.processingHeaderObj.nWindows != 0:
756 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
756 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
757 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - \
757 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - \
758 self.dataOut.heightList[0]
758 self.dataOut.heightList[0]
759 self.processingHeaderObj.nHeights = self.dataOut.nHeights
759 self.processingHeaderObj.nHeights = self.dataOut.nHeights
760 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
760 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
761
761
762 self.processingHeaderObj.processFlags = self.getProcessFlags()
762 self.processingHeaderObj.processFlags = self.getProcessFlags()
763
763
764 self.setBasicHeader()
764 self.setBasicHeader()
765 No newline at end of file
765
Show file before | Show file after | Hide comments
@@ -1,367 +1,367
1 '''
1 '''
2 Updated for multiprocessing
2 Updated for multiprocessing
3 Author : Sergio Cortez
3 Author : Sergio Cortez
4 Jan 2018
4 Jan 2018
5 Abstract:
5 Abstract:
6 Base class for processing units and operations. A decorator provides multiprocessing features and interconnect the processes created.
6 Base class for processing units and operations. A decorator provides multiprocessing features and interconnect the processes created.
7 The argument (kwargs) sent from the controller is parsed and filtered via the decorator for each processing unit or operation instantiated.
7 The argument (kwargs) sent from the controller is parsed and filtered via the decorator for each processing unit or operation instantiated.
8 The decorator handle also the methods inside the processing unit to be called from the main script (not as operations) (OPERATION -> type ='self').
8 The decorator handle also the methods inside the processing unit to be called from the main script (not as operations) (OPERATION -> type ='self').
9
9
10 Based on:
10 Based on:
11 $Author: murco $
11 $Author: murco $
12 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
12 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
13 '''
13 '''
14 from platform import python_version
14 from platform import python_version
15 import inspect
15 import inspect
16 import zmq
16 import zmq
17 import time
17 import time
18 import pickle
18 import pickle
19 import os
19 import os
20 from multiprocessing import Process
20 from multiprocessing import Process
21 from schainpy.utils import log
21 from schainpy.utils import log
22
22
23
23
24 class ProcessingUnit(object):
24 class ProcessingUnit(object):
25
25
26 """
26 """
27 Update - Jan 2018 - MULTIPROCESSING
27 Update - Jan 2018 - MULTIPROCESSING
28 All the "call" methods present in the previous base were removed.
28 All the "call" methods present in the previous base were removed.
29 The majority of operations are independant processes, thus
29 The majority of operations are independant processes, thus
30 the decorator is in charge of communicate the operation processes
30 the decorator is in charge of communicate the operation processes
31 with the proccessing unit via IPC.
31 with the proccessing unit via IPC.
32
32
33 The constructor does not receive any argument. The remaining methods
33 The constructor does not receive any argument. The remaining methods
34 are related with the operations to execute.
34 are related with the operations to execute.
35
35
36
36
37 """
37 """
38
38
39 METHODS = {}
39 METHODS = {}
40 dataIn = None
40 dataIn = None
41 dataInList = []
41 dataInList = []
42 id = None
42 id = None
43 inputId = None
43 inputId = None
44 dataOut = None
44 dataOut = None
45 dictProcs = None
45 dictProcs = None
46 isConfig = False
46 isConfig = False
47
47
48 def __init__(self):
48 def __init__(self):
49
49
50 self.dataIn = None
50 self.dataIn = None
51 self.dataOut = None
51 self.dataOut = None
52 self.isConfig = False
52 self.isConfig = False
53 self.operations = []
53 self.operations = []
54
54
55 def getAllowedArgs(self):
55 def getAllowedArgs(self):
56 if hasattr(self, '__attrs__'):
56 if hasattr(self, '__attrs__'):
57 return self.__attrs__
57 return self.__attrs__
58 else:
58 else:
59 return inspect.getargspec(self.run).args
59 return inspect.getargspec(self.run).args
60
60
61 def addOperation(self, conf, operation):
61 def addOperation(self, conf, operation):
62
62
63 """
63 """
64 This method is used in the controller, and update the dictionary containing the operations to execute. The dict
64 This method is used in the controller, and update the dictionary containing the operations to execute. The dict
65 posses the id of the operation process (IPC purposes)
65 posses the id of the operation process (IPC purposes)
66
66
67 Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
67 Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
68 identificador asociado a este objeto.
68 identificador asociado a este objeto.
69
69
70 Input:
70 Input:
71
71
72 object : objeto de la clase "Operation"
72 object : objeto de la clase "Operation"
73
73
74 Return:
74 Return:
75
75
76 objId : identificador del objeto, necesario para comunicar con master(procUnit)
76 objId : identificador del objeto, necesario para comunicar con master(procUnit)
77 """
77 """
78
78
79 self.operations.append((operation, conf.type, conf.id, conf.getKwargs()))
79 self.operations.append((operation, conf.type, conf.id, conf.getKwargs()))
80
80
81 def getOperationObj(self, objId):
81 def getOperationObj(self, objId):
82
82
83 if objId not in list(self.operations.keys()):
83 if objId not in list(self.operations.keys()):
84 return None
84 return None
85
85
86 return self.operations[objId]
86 return self.operations[objId]
87
87
88 def operation(self, **kwargs):
88 def operation(self, **kwargs):
89
89
90 """
90 """
91 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
91 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
92 atributos del objeto dataOut
92 atributos del objeto dataOut
93
93
94 Input:
94 Input:
95
95
96 **kwargs : Diccionario de argumentos de la funcion a ejecutar
96 **kwargs : Diccionario de argumentos de la funcion a ejecutar
97 """
97 """
98
98
99 raise NotImplementedError
99 raise NotImplementedError
100
100
101 def setup(self):
101 def setup(self):
102
102
103 raise NotImplementedError
103 raise NotImplementedError
104
104
105 def run(self):
105 def run(self):
106
106
107 raise NotImplementedError
107 raise NotImplementedError
108
108
109 def close(self):
109 def close(self):
110 #Close every thread, queue or any other object here is it is neccesary.
110 #Close every thread, queue or any other object here is it is neccesary.
111 return
111 return
112
112
113 class Operation(object):
113 class Operation(object):
114
114
115 """
115 """
116 Update - Jan 2018 - MULTIPROCESSING
116 Update - Jan 2018 - MULTIPROCESSING
117
117
118 Most of the methods remained the same. The decorator parse the arguments and executed the run() method for each process.
118 Most of the methods remained the same. The decorator parse the arguments and executed the run() method for each process.
119 The constructor doe snot receive any argument, neither the baseclass.
119 The constructor doe snot receive any argument, neither the baseclass.
120
120
121
121
122 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
122 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
123 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
123 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
124 acumulacion dentro de esta clase
124 acumulacion dentro de esta clase
125
125
126 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
126 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
127
127
128 """
128 """
129 id = None
129 id = None
130 __buffer = None
130 __buffer = None
131 dest = None
131 dest = None
132 isConfig = False
132 isConfig = False
133 readyFlag = None
133 readyFlag = None
134
134
135 def __init__(self):
135 def __init__(self):
136
136
137 self.buffer = None
137 self.buffer = None
138 self.dest = None
138 self.dest = None
139 self.isConfig = False
139 self.isConfig = False
140 self.readyFlag = False
140 self.readyFlag = False
141
141
142 if not hasattr(self, 'name'):
142 if not hasattr(self, 'name'):
143 self.name = self.__class__.__name__
143 self.name = self.__class__.__name__
144
144
145 def getAllowedArgs(self):
145 def getAllowedArgs(self):
146 if hasattr(self, '__attrs__'):
146 if hasattr(self, '__attrs__'):
147 return self.__attrs__
147 return self.__attrs__
148 else:
148 else:
149 return inspect.getargspec(self.run).args
149 return inspect.getargspec(self.run).args
150
150
151 def setup(self):
151 def setup(self):
152
152
153 self.isConfig = True
153 self.isConfig = True
154
154
155 raise NotImplementedError
155 raise NotImplementedError
156
156
157
157
158 def run(self, dataIn, **kwargs):
158 def run(self, dataIn, **kwargs):
159
159
160 """
160 """
161 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
161 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
162 atributos del objeto dataIn.
162 atributos del objeto dataIn.
163
163
164 Input:
164 Input:
165
165
166 dataIn : objeto del tipo JROData
166 dataIn : objeto del tipo JROData
167
167
168 Return:
168 Return:
169
169
170 None
170 None
171
171
172 Affected:
172 Affected:
173 __buffer : buffer de recepcion de datos.
173 __buffer : buffer de recepcion de datos.
174
174
175 """
175 """
176 if not self.isConfig:
176 if not self.isConfig:
177 self.setup(**kwargs)
177 self.setup(**kwargs)
178
178
179 raise NotImplementedError
179 raise NotImplementedError
180
180
181 def close(self):
181 def close(self):
182
182
183 pass
183 pass
184
184
185
185
186 def MPDecorator(BaseClass):
186 def MPDecorator(BaseClass):
187
187
188 """
188 """
189 Multiprocessing class decorator
189 Multiprocessing class decorator
190
190
191 This function add multiprocessing features to a BaseClass. Also, it handle
191 This function add multiprocessing features to a BaseClass. Also, it handle
192 the communication beetween processes (readers, procUnits and operations).
192 the communication beetween processes (readers, procUnits and operations).
193 """
193 """
194
194
195 class MPClass(BaseClass, Process):
195 class MPClass(BaseClass, Process):
196
196
197 def __init__(self, *args, **kwargs):
197 def __init__(self, *args, **kwargs):
198 super(MPClass, self).__init__()
198 super(MPClass, self).__init__()
199 Process.__init__(self)
199 Process.__init__(self)
200 self.operationKwargs = {}
200 self.operationKwargs = {}
201 self.args = args
201 self.args = args
202 self.kwargs = kwargs
202 self.kwargs = kwargs
203 self.sender = None
203 self.sender = None
204 self.receiver = None
204 self.receiver = None
205 self.name = BaseClass.__name__
205 self.name = BaseClass.__name__
206
206
207 if len(self.args) is 3:
207 if len(self.args) is 3:
208 self.typeProc = "ProcUnit"
208 self.typeProc = "ProcUnit"
209 self.id = args[0]
209 self.id = args[0]
210 self.inputId = args[1]
210 self.inputId = args[1]
211 self.project_id = args[2]
211 self.project_id = args[2]
212 else:
212 else:
213 self.id = args[0]
213 self.id = args[0]
214 self.inputId = args[0]
214 self.inputId = args[0]
215 self.project_id = args[1]
215 self.project_id = args[1]
216 self.typeProc = "Operation"
216 self.typeProc = "Operation"
217
217
218 def getAllowedArgs(self):
218 def getAllowedArgs(self):
219
219
220 if hasattr(self, '__attrs__'):
220 if hasattr(self, '__attrs__'):
221 return self.__attrs__
221 return self.__attrs__
222 else:
222 else:
223 return inspect.getargspec(BaseClass.run).args
223 return inspect.getargspec(BaseClass.run).args
224
224
225 def subscribe(self):
225 def subscribe(self):
226 '''
226 '''
227 This function create a socket to receive objects from the
227 This function create a socket to receive objects from the
228 topic `inputId`.
228 topic `inputId`.
229 '''
229 '''
230
230
231 c = zmq.Context()
231 c = zmq.Context()
232 self.receiver = c.socket(zmq.SUB)
232 self.receiver = c.socket(zmq.SUB)
233 self.receiver.connect('ipc:///tmp/schain/{}_pub'.format(self.project_id))
233 self.receiver.connect('ipc:///tmp/schain/{}_pub'.format(self.project_id))
234 self.receiver.setsockopt(zmq.SUBSCRIBE, self.inputId.encode())
234 self.receiver.setsockopt(zmq.SUBSCRIBE, self.inputId.encode())
235
235
236 def listen(self):
236 def listen(self):
237 '''
237 '''
238 This function waits for objects and deserialize using pickle
238 This function waits for objects and deserialize using pickle
239 '''
239 '''
240
240
241 data = pickle.loads(self.receiver.recv_multipart()[1])
241 data = pickle.loads(self.receiver.recv_multipart()[1])
242 return data
242 return data
243
243
244 def set_publisher(self):
244 def set_publisher(self):
245 '''
245 '''
246 This function create a socket for publishing purposes.
246 This function create a socket for publishing purposes.
247 '''
247 '''
248
248
249 time.sleep(1)
249 time.sleep(1)
250 c = zmq.Context()
250 c = zmq.Context()
251 self.sender = c.socket(zmq.PUB)
251 self.sender = c.socket(zmq.PUB)
252 self.sender.connect('ipc:///tmp/schain/{}_sub'.format(self.project_id))
252 self.sender.connect('ipc:///tmp/schain/{}_sub'.format(self.project_id))
253
253
254 def publish(self, data, id):
254 def publish(self, data, id):
255 '''
255 '''
256 This function publish an object, to a specific topic.
256 This function publish an object, to a specific topic.
257 '''
257 '''
258
258
259 self.sender.send_multipart([str(id).encode(), pickle.dumps(data)])
259 self.sender.send_multipart([str(id).encode(), pickle.dumps(data)])
260
260
261 def runReader(self):
261 def runReader(self):
262 '''
262 '''
263 Run fuction for read units
263 Run fuction for read units
264 '''
264 '''
265 while True:
265 while True:
266
266
267 BaseClass.run(self, **self.kwargs)
267 BaseClass.run(self, **self.kwargs)
268
268
269 if self.dataOut.error[0] == -1:
269 if self.dataOut.error[0] == -1:
270 log.error(self.dataOut.error[1])
270 log.error(self.dataOut.error[1])
271 self.publish('end', self.id)
271 self.publish('end', self.id)
272 #self.sender.send_multipart([str(self.project_id).encode(), 'end'.encode()])
272 #self.sender.send_multipart([str(self.project_id).encode(), 'end'.encode()])
273 break
273 break
274
274
275 for op, optype, id, kwargs in self.operations:
275 for op, optype, id, kwargs in self.operations:
276 if optype=='self':
276 if optype=='self':
277 op(**kwargs)
277 op(**kwargs)
278 elif optype=='other':
278 elif optype=='other':
279 self.dataOut = op.run(self.dataOut, **self.kwargs)
279 self.dataOut = op.run(self.dataOut, **self.kwargs)
280 elif optype=='external':
280 elif optype=='external':
281 self.publish(self.dataOut, opId)
281 self.publish(self.dataOut, opId)
282
282
283 if self.dataOut.flagNoData:
283 if self.dataOut.flagNoData:
284 continue
284 continue
285
285
286 self.publish(self.dataOut, self.id)
286 self.publish(self.dataOut, self.id)
287
287
288 def runProc(self):
288 def runProc(self):
289 '''
289 '''
290 Run function for proccessing units
290 Run function for proccessing units
291 '''
291 '''
292
292
293 while True:
293 while True:
294 self.dataIn = self.listen()
294 self.dataIn = self.listen()
295
295
296 if self.dataIn == 'end':
296 if self.dataIn == 'end':
297 self.publish('end', self.id)
297 self.publish('end', self.id)
298 for op, optype, opId, kwargs in self.operations:
298 for op, optype, opId, kwargs in self.operations:
299 if optype == 'external':
299 if optype == 'external':
300 self.publish('end', opId)
300 self.publish('end', opId)
301 break
301 break
302
302
303 if self.dataIn.flagNoData:
303 if self.dataIn.flagNoData:
304 continue
304 continue
305
305
306 BaseClass.run(self, **self.kwargs)
306 BaseClass.run(self, **self.kwargs)
307
307
308 for op, optype, opId, kwargs in self.operations:
308 for op, optype, opId, kwargs in self.operations:
309 if optype=='self':
309 if optype=='self':
310 op(**kwargs)
310 op(**kwargs)
311 elif optype=='other':
311 elif optype=='other':
312 self.dataOut = op.run(self.dataOut, **kwargs)
312 self.dataOut = op.run(self.dataOut, **kwargs)
313 elif optype=='external':
313 elif optype=='external':
314 self.publish(self.dataOut, opId)
314 self.publish(self.dataOut, opId)
315
315
316 if self.dataOut.flagNoData:
316 if self.dataOut.flagNoData:
317 continue
317 continue
318
318
319 self.publish(self.dataOut, self.id)
319 self.publish(self.dataOut, self.id)
320
320
321 def runOp(self):
321 def runOp(self):
322 '''
322 '''
323 Run function for operations
323 Run function for operations
324 '''
324 '''
325
325
326 while True:
326 while True:
327
327
328 dataOut = self.listen()
328 dataOut = self.listen()
329
329
330 if dataOut == 'end':
330 if dataOut == 'end':
331 break
331 break
332
332
333 BaseClass.run(self, dataOut, **self.kwargs)
333 BaseClass.run(self, dataOut, **self.kwargs)
334
334
335 def run(self):
335 def run(self):
336
336
337 if self.typeProc is "ProcUnit":
337 if self.typeProc is "ProcUnit":
338
338
339 if self.inputId is not None:
339 if self.inputId is not None:
340 self.subscribe()
340 self.subscribe()
341 self.set_publisher()
341 self.set_publisher()
342
342
343 if 'Reader' not in BaseClass.__name__:
343 if 'Reader' not in BaseClass.__name__:
344 self.runProc()
344 self.runProc()
345 else:
345 else:
346 self.runReader()
346 self.runReader()
347
347
348 elif self.typeProc is "Operation":
348 elif self.typeProc is "Operation":
349
349
350 self.subscribe()
350 self.subscribe()
351 self.runOp()
351 self.runOp()
352
352
353 else:
353 else:
354 raise ValueError("Unknown type")
354 raise ValueError("Unknown type")
355
355
356 print("%s done" % BaseClass.__name__)
356 print("%s done" % BaseClass.__name__)
357 self.close()
357 self.close()
358
358
359 def close(self):
359 def close(self):
360
360
361 if self.sender:
361 if self.sender:
362 self.sender.close()
362 self.sender.close()
363
363
364 if self.receiver:
364 if self.receiver:
365 self.receiver.close()
365 self.receiver.close()
366
366
367 return MPClass No newline at end of file
367 return MPClass
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
General Comments 0
You need to be logged in to leave comments. Login now