##// END OF EJS Templates
schain
RSS
grafico outliers, grafico de integraciones, funciones de obtención ruído, etc
joabAM -
r1540:c8092aa564c5
parent child
Show More
Side by Side Unified
Context file:
r1540:c8092aa564c5 Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -1,126 +1,126
1 1 #include <Python.h>
2 2 #include <numpy/arrayobject.h>
3 3 #include <math.h>
4 4
5 5
6 6 static PyObject *hildebrand_sekhon(PyObject *self, PyObject *args) {
7 7 double navg;
8 8 PyObject *data_obj, *data_array;
9 9
10 10 if (!PyArg_ParseTuple(args, "Od", &data_obj, &navg)) {
11 11 return NULL;
12 12 }
13 13
14 14 data_array = PyArray_FROM_OTF(data_obj, NPY_FLOAT64, NPY_IN_ARRAY);
15 15
16 16 if (data_array == NULL) {
17 17 Py_XDECREF(data_array);
18 18 Py_XDECREF(data_obj);
19 19 return NULL;
20 20 }
21 21 double *sortdata = (double*)PyArray_DATA(data_array);
22 22 int lenOfData = (int)PyArray_SIZE(data_array) ;
23 double nums_min = lenOfData*0.2;
23 double nums_min = lenOfData*0.5;//0.2
24 24 if (nums_min <= 5) nums_min = 5;
25 25 double sump = 0;
26 26 double sumq = 0;
27 27 long j = 0;
28 28 int cont = 1;
29 29 double rtest = 0;
30 30 while ((cont == 1) && (j < lenOfData)) {
31 31 sump = sump + sortdata[j];
32 32 sumq = sumq + pow(sortdata[j], 2);
33 33 if (j > nums_min) {
34 34 rtest = (double)j/(j-1) + 1/navg;
35 35 if ((sumq*j) > (rtest*pow(sump, 2))) {
36 36 j = j - 1;
37 37 sump = sump - sortdata[j];
38 38 sumq = sumq - pow(sortdata[j],2);
39 39 cont = 0;
40 40 }
41 41 }
42 42 j = j + 1;
43 43 }
44 44
45 45 double lnoise = sump / j;
46 46
47 47 Py_DECREF(data_array);
48 48
49 49 // return PyLong_FromLong(lnoise);
50 50 return PyFloat_FromDouble(lnoise);
51 51 }
52 52
53 53 static PyObject *hildebrand_sekhon2(PyObject *self, PyObject *args) {
54 54 double navg;
55 55 PyObject *data_obj, *data_array;
56 56
57 57 if (!PyArg_ParseTuple(args, "Od", &data_obj, &navg)) {
58 58 return NULL;
59 59 }
60 60 data_array = PyArray_FROM_OTF(data_obj, NPY_FLOAT64, NPY_IN_ARRAY);
61 61
62 62 if (data_array == NULL) {
63 63 Py_XDECREF(data_array);
64 64 Py_XDECREF(data_obj);
65 65 return NULL;
66 66 }
67 67 double *sortdata = (double*)PyArray_DATA(data_array);
68 68 int lenOfData = (int)PyArray_SIZE(data_array) ;
69 69 double nums_min = lenOfData*0.85;
70 70 if (nums_min <= 5) nums_min = 5;
71 71 double sump = 0;
72 72 double sumq = 0;
73 73 long j = 0;
74 74 int cont = 1;
75 75 double rtest = 0;
76 76 while ((cont == 1) && (j < lenOfData)) {
77 77 sump = sump + sortdata[j];
78 78 sumq = sumq + pow(sortdata[j], 2);
79 79 if (j > nums_min) {
80 80 rtest = (double)j/(j-1) + 1/navg;
81 81 if ((sumq*j) > (rtest*pow(sump, 2))) {
82 82 j = j - 1;
83 83 sump = sump - sortdata[j];
84 84 sumq = sumq - pow(sortdata[j],2);
85 85 cont = 0;
86 86 }
87 87 }
88 88 j = j + 1;
89 89 }
90 90
91 91 Py_DECREF(data_array);
92 92
93 93 return PyLong_FromLong(j);
94 94 }
95 95
96 96
97 97 static PyMethodDef noiseMethods[] = {
98 98 { "hildebrand_sekhon", hildebrand_sekhon, METH_VARARGS, "Get noise with hildebrand_sekhon algorithm" },
99 99 { "hildebrand_sekhon2", hildebrand_sekhon2, METH_VARARGS, "Variation for satellite cleaning" },
100 100 { NULL, NULL, 0, NULL }
101 101 };
102 102
103 103 #if PY_MAJOR_VERSION >= 3
104 104
105 105 static struct PyModuleDef noisemodule = {
106 106 PyModuleDef_HEAD_INIT,
107 107 "_noise",
108 108 "Get noise with hildebrand_sekhon algorithm",
109 109 -1,
110 110 noiseMethods
111 111 };
112 112
113 113 #endif
114 114
115 115 #if PY_MAJOR_VERSION >= 3
116 116 PyMODINIT_FUNC PyInit__noise(void) {
117 117 Py_Initialize();
118 118 import_array();
119 119 return PyModule_Create(&noisemodule);
120 120 }
121 121 #else
122 122 PyMODINIT_FUNC init_noise() {
123 123 Py_InitModule("_noise", noiseMethods);
124 124 import_array();
125 125 }
126 126 #endif
Show file before | Show file after | Hide comments
@@ -1,685 +1,686
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """API to create signal chain projects
6 6
7 7 The API is provide through class: Project
8 8 """
9 9
10 10 import re
11 11 import sys
12 12 import ast
13 13 import datetime
14 14 import traceback
15 15 import time
16 16 import multiprocessing
17 17 from multiprocessing import Process, Queue
18 18 from threading import Thread
19 19 from xml.etree.ElementTree import ElementTree, Element, SubElement
20 20
21 21 from schainpy.admin import Alarm, SchainWarning
22 22 from schainpy.model import *
23 23 from schainpy.utils import log
24 24
25 25 if 'darwin' in sys.platform and sys.version_info[0] == 3 and sys.version_info[1] > 7:
26 26 multiprocessing.set_start_method('fork')
27 27
28 28 class ConfBase():
29 29
30 30 def __init__(self):
31 31
32 32 self.id = '0'
33 33 self.name = None
34 34 self.priority = None
35 35 self.parameters = {}
36 36 self.object = None
37 37 self.operations = []
38 38
39 39 def getId(self):
40 40
41 41 return self.id
42 42
43 43 def getNewId(self):
44 44
45 45 return int(self.id) * 10 + len(self.operations) + 1
46 46
47 47 def updateId(self, new_id):
48 48
49 49 self.id = str(new_id)
50 50
51 51 n = 1
52 52 for conf in self.operations:
53 53 conf_id = str(int(new_id) * 10 + n)
54 54 conf.updateId(conf_id)
55 55 n += 1
56 56
57 57 def getKwargs(self):
58 58
59 59 params = {}
60 60
61 61 for key, value in self.parameters.items():
62 62 if value not in (None, '', ' '):
63 63 params[key] = value
64 64
65 65 return params
66 66
67 67 def update(self, **kwargs):
68 68
69 69 for key, value in kwargs.items():
70 70 self.addParameter(name=key, value=value)
71 71
72 72 def addParameter(self, name, value, format=None):
73 73 '''
74 74 '''
75 75
76 76 if isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
77 77 self.parameters[name] = datetime.date(*[int(x) for x in value.split('/')])
78 78 elif isinstance(value, str) and re.search(r'(\d+:\d+:\d+)', value):
79 79 self.parameters[name] = datetime.time(*[int(x) for x in value.split(':')])
80 80 else:
81 81 try:
82 82 self.parameters[name] = ast.literal_eval(value)
83 83 except:
84 84 if isinstance(value, str) and ',' in value:
85 85 self.parameters[name] = value.split(',')
86 86 else:
87 87 self.parameters[name] = value
88 88
89 89 def getParameters(self):
90 90
91 91 params = {}
92 92 for key, value in self.parameters.items():
93 93 s = type(value).__name__
94 94 if s == 'date':
95 95 params[key] = value.strftime('%Y/%m/%d')
96 96 elif s == 'time':
97 97 params[key] = value.strftime('%H:%M:%S')
98 98 else:
99 99 params[key] = str(value)
100 100
101 101 return params
102 102
103 103 def makeXml(self, element):
104 104
105 105 xml = SubElement(element, self.ELEMENTNAME)
106 106 for label in self.xml_labels:
107 107 xml.set(label, str(getattr(self, label)))
108 108
109 109 for key, value in self.getParameters().items():
110 110 xml_param = SubElement(xml, 'Parameter')
111 111 xml_param.set('name', key)
112 112 xml_param.set('value', value)
113 113
114 114 for conf in self.operations:
115 115 conf.makeXml(xml)
116 116
117 117 def __str__(self):
118 118
119 119 if self.ELEMENTNAME == 'Operation':
120 120 s = ' {}[id={}]\n'.format(self.name, self.id)
121 121 else:
122 122 s = '{}[id={}, inputId={}]\n'.format(self.name, self.id, self.inputId)
123 123
124 124 for key, value in self.parameters.items():
125 125 if self.ELEMENTNAME == 'Operation':
126 126 s += ' {}: {}\n'.format(key, value)
127 127 else:
128 128 s += ' {}: {}\n'.format(key, value)
129 129
130 130 for conf in self.operations:
131 131 s += str(conf)
132 132
133 133 return s
134 134
135 135 class OperationConf(ConfBase):
136 136
137 137 ELEMENTNAME = 'Operation'
138 138 xml_labels = ['id', 'name']
139 139
140 140 def setup(self, id, name, priority, project_id, err_queue):
141 141
142 142 self.id = str(id)
143 143 self.project_id = project_id
144 144 self.name = name
145 145 self.type = 'other'
146 146 self.err_queue = err_queue
147 147
148 148 def readXml(self, element, project_id, err_queue):
149 149
150 150 self.id = element.get('id')
151 151 self.name = element.get('name')
152 152 self.type = 'other'
153 153 self.project_id = str(project_id)
154 154 self.err_queue = err_queue
155 155
156 156 for elm in element.iter('Parameter'):
157 157 self.addParameter(elm.get('name'), elm.get('value'))
158 158
159 159 def createObject(self):
160 160
161 161 className = eval(self.name)
162 162
163 163 if 'Plot' in self.name or 'Writer' in self.name or 'Send' in self.name or 'print' in self.name:
164 164 kwargs = self.getKwargs()
165 165 opObj = className(self.name, self.id, self.project_id, self.err_queue, **kwargs)
166 166 opObj.start()
167 167 self.type = 'external'
168 168 else:
169 169 opObj = className()
170 170
171 171 self.object = opObj
172 172 return opObj
173 173
174 174 class ProcUnitConf(ConfBase):
175 175
176 176 ELEMENTNAME = 'ProcUnit'
177 177 xml_labels = ['id', 'inputId', 'name']
178 178
179 179 def setup(self, project_id, id, name, datatype, inputId, err_queue):
180 180 '''
181 181 '''
182 182
183 183 if datatype == None and name == None:
184 184 raise ValueError('datatype or name should be defined')
185 185
186 186 if name == None:
187 187 if 'Proc' in datatype:
188 188 name = datatype
189 189 else:
190 190 name = '%sProc' % (datatype)
191 191
192 192 if datatype == None:
193 193 datatype = name.replace('Proc', '')
194 194
195 195 self.id = str(id)
196 196 self.project_id = project_id
197 197 self.name = name
198 198 self.datatype = datatype
199 199 self.inputId = inputId
200 200 self.err_queue = err_queue
201 201 self.operations = []
202 202 self.parameters = {}
203 203
204 204 def removeOperation(self, id):
205 205
206 206 i = [1 if x.id==id else 0 for x in self.operations]
207 207 self.operations.pop(i.index(1))
208 208
209 209 def getOperation(self, id):
210 210
211 211 for conf in self.operations:
212 212 if conf.id == id:
213 213 return conf
214 214
215 215 def addOperation(self, name, optype='self'):
216 216 '''
217 217 '''
218 218
219 219 id = self.getNewId()
220 220 conf = OperationConf()
221 221 conf.setup(id, name=name, priority='0', project_id=self.project_id, err_queue=self.err_queue)
222 222 self.operations.append(conf)
223 223
224 224 return conf
225 225
226 226 def readXml(self, element, project_id, err_queue):
227 227
228 228 self.id = element.get('id')
229 229 self.name = element.get('name')
230 230 self.inputId = None if element.get('inputId') == 'None' else element.get('inputId')
231 231 self.datatype = element.get('datatype', self.name.replace(self.ELEMENTNAME.replace('Unit', ''), ''))
232 232 self.project_id = str(project_id)
233 233 self.err_queue = err_queue
234 234 self.operations = []
235 235 self.parameters = {}
236 236
237 237 for elm in element:
238 238 if elm.tag == 'Parameter':
239 239 self.addParameter(elm.get('name'), elm.get('value'))
240 240 elif elm.tag == 'Operation':
241 241 conf = OperationConf()
242 242 conf.readXml(elm, project_id, err_queue)
243 243 self.operations.append(conf)
244 244
245 245 def createObjects(self):
246 246 '''
247 247 Instancia de unidades de procesamiento.
248 248 '''
249 249
250 250 className = eval(self.name)
251 251 kwargs = self.getKwargs()
252 252 procUnitObj = className()
253 253 procUnitObj.name = self.name
254 254 log.success('creating process... id: {}, inputId: {}'.format(self.id,self.inputId), self.name)
255 255
256 256 for conf in self.operations:
257 257
258 258 opObj = conf.createObject()
259 259
260 260 log.success('adding operation: {}, type:{}'.format(conf.name,conf.type), self.name)
261 261
262 262 procUnitObj.addOperation(conf, opObj)
263 263
264 264 self.object = procUnitObj
265 265
266 266 def run(self):
267 267 '''
268 268 '''
269 269 return self.object.call(**self.getKwargs())
270 270
271 271 class ReadUnitConf(ProcUnitConf):
272 272
273 273 ELEMENTNAME = 'ReadUnit'
274 274
275 275 def __init__(self):
276 276
277 277 self.id = None
278 278 self.datatype = None
279 279 self.name = None
280 280 self.inputId = None
281 281 self.operations = []
282 282 self.parameters = {}
283 283
284 284 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
285 285 startTime='', endTime='', server=None, **kwargs):
286 286
287 287 if datatype == None and name == None:
288 288 raise ValueError('datatype or name should be defined')
289 289 if name == None:
290 290 if 'Reader' in datatype:
291 291 name = datatype
292 292 datatype = name.replace('Reader','')
293 293 else:
294 294 name = '{}Reader'.format(datatype)
295 295 if datatype == None:
296 296 if 'Reader' in name:
297 297 datatype = name.replace('Reader','')
298 298 else:
299 299 datatype = name
300 300 name = '{}Reader'.format(name)
301 301
302 302 self.id = id
303 303 self.project_id = project_id
304 304 self.name = name
305 305 self.datatype = datatype
306 306 self.err_queue = err_queue
307 307
308 308 self.addParameter(name='path', value=path)
309 309 self.addParameter(name='startDate', value=startDate)
310 310 self.addParameter(name='endDate', value=endDate)
311 311 self.addParameter(name='startTime', value=startTime)
312 312 self.addParameter(name='endTime', value=endTime)
313 313
314 314 for key, value in kwargs.items():
315 315 self.addParameter(name=key, value=value)
316 316
317 317
318 318 class Project(Process):
319 319 """API to create signal chain projects"""
320 320
321 321 ELEMENTNAME = 'Project'
322 322
323 323 def __init__(self, name=''):
324 324
325 325 Process.__init__(self)
326 326 self.id = '1'
327 327 if name:
328 328 self.name = '{} ({})'.format(Process.__name__, name)
329 329 self.filename = None
330 330 self.description = None
331 331 self.email = None
332 332 self.alarm = []
333 333 self.configurations = {}
334 334 # self.err_queue = Queue()
335 335 self.err_queue = None
336 336 self.started = False
337 337
338 338 def getNewId(self):
339 339
340 340 idList = list(self.configurations.keys())
341 341 id = int(self.id) * 10
342 342
343 while True:
343 while 1:
344 344 id += 1
345 345
346 346 if str(id) in idList:
347 347 continue
348 348
349 349 break
350 350
351 351 return str(id)
352 352
353 353 def updateId(self, new_id):
354 354
355 355 self.id = str(new_id)
356 356
357 357 keyList = list(self.configurations.keys())
358 358 keyList.sort()
359 359
360 360 n = 1
361 361 new_confs = {}
362 362
363 363 for procKey in keyList:
364 364
365 365 conf = self.configurations[procKey]
366 366 idProcUnit = str(int(self.id) * 10 + n)
367 367 conf.updateId(idProcUnit)
368 368 new_confs[idProcUnit] = conf
369 369 n += 1
370 370
371 371 self.configurations = new_confs
372 372
373 373 def setup(self, id=1, name='', description='', email=None, alarm=[]):
374 374
375 375 self.id = str(id)
376 376 self.description = description
377 377 self.email = email
378 378 self.alarm = alarm
379 379 if name:
380 380 self.name = '{} ({})'.format(Process.__name__, name)
381 381
382 382 def update(self, **kwargs):
383 383
384 384 for key, value in kwargs.items():
385 385 setattr(self, key, value)
386 386
387 387 def clone(self):
388 388
389 389 p = Project()
390 390 p.id = self.id
391 391 p.name = self.name
392 392 p.description = self.description
393 393 p.configurations = self.configurations.copy()
394 394
395 395 return p
396 396
397 397 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
398 398
399 399 '''
400 400 '''
401 401
402 402 if id is None:
403 403 idReadUnit = self.getNewId()
404 404 else:
405 405 idReadUnit = str(id)
406 406
407 407 conf = ReadUnitConf()
408 408 conf.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
409 409 self.configurations[conf.id] = conf
410 410
411 411 return conf
412 412
413 413 def addProcUnit(self, id=None, inputId='0', datatype=None, name=None):
414 414
415 415 '''
416 416 '''
417 417
418 418 if id is None:
419 419 idProcUnit = self.getNewId()
420 420 else:
421 421 idProcUnit = id
422 422
423 423 conf = ProcUnitConf()
424 424 conf.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
425 425 self.configurations[conf.id] = conf
426 426
427 427 return conf
428 428
429 429 def removeProcUnit(self, id):
430 430
431 431 if id in self.configurations:
432 432 self.configurations.pop(id)
433 433
434 434 def getReadUnit(self):
435 435
436 436 for obj in list(self.configurations.values()):
437 437 if obj.ELEMENTNAME == 'ReadUnit':
438 438 return obj
439 439
440 440 return None
441 441
442 442 def getProcUnit(self, id):
443 443
444 444 return self.configurations[id]
445 445
446 446 def getUnits(self):
447 447
448 448 keys = list(self.configurations)
449 449 keys.sort()
450 450
451 451 for key in keys:
452 452 yield self.configurations[key]
453 453
454 454 def updateUnit(self, id, **kwargs):
455 455
456 456 conf = self.configurations[id].update(**kwargs)
457 457
458 458 def makeXml(self):
459 459
460 460 xml = Element('Project')
461 461 xml.set('id', str(self.id))
462 462 xml.set('name', self.name)
463 463 xml.set('description', self.description)
464 464
465 465 for conf in self.configurations.values():
466 466 conf.makeXml(xml)
467 467
468 468 self.xml = xml
469 469
470 470 def writeXml(self, filename=None):
471 471
472 472 if filename == None:
473 473 if self.filename:
474 474 filename = self.filename
475 475 else:
476 476 filename = 'schain.xml'
477 477
478 478 if not filename:
479 479 print('filename has not been defined. Use setFilename(filename) for do it.')
480 480 return 0
481 481
482 482 abs_file = os.path.abspath(filename)
483 483
484 484 if not os.access(os.path.dirname(abs_file), os.W_OK):
485 485 print('No write permission on %s' % os.path.dirname(abs_file))
486 486 return 0
487 487
488 488 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
489 489 print('File %s already exists and it could not be overwriten' % abs_file)
490 490 return 0
491 491
492 492 self.makeXml()
493 493
494 494 ElementTree(self.xml).write(abs_file, method='xml')
495 495
496 496 self.filename = abs_file
497 497
498 498 return 1
499 499
500 500 def readXml(self, filename):
501 501
502 502 abs_file = os.path.abspath(filename)
503 503
504 504 self.configurations = {}
505 505
506 506 try:
507 507 self.xml = ElementTree().parse(abs_file)
508 508 except:
509 509 log.error('Error reading %s, verify file format' % filename)
510 510 return 0
511 511
512 512 self.id = self.xml.get('id')
513 513 self.name = self.xml.get('name')
514 514 self.description = self.xml.get('description')
515 515
516 516 for element in self.xml:
517 517 if element.tag == 'ReadUnit':
518 518 conf = ReadUnitConf()
519 519 conf.readXml(element, self.id, self.err_queue)
520 520 self.configurations[conf.id] = conf
521 521 elif element.tag == 'ProcUnit':
522 522 conf = ProcUnitConf()
523 523 input_proc = self.configurations[element.get('inputId')]
524 524 conf.readXml(element, self.id, self.err_queue)
525 525 self.configurations[conf.id] = conf
526 526
527 527 self.filename = abs_file
528 528
529 529 return 1
530 530
531 531 def __str__(self):
532 532
533 533 text = '\nProject[id=%s, name=%s, description=%s]\n\n' % (
534 534 self.id,
535 535 self.name,
536 536 self.description,
537 537 )
538 538
539 539 for conf in self.configurations.values():
540 540 text += '{}'.format(conf)
541 541
542 542 return text
543 543
544 544 def createObjects(self):
545 545
546 546 keys = list(self.configurations.keys())
547 547 keys.sort()
548 548 for key in keys:
549 549 conf = self.configurations[key]
550 550 conf.createObjects()
551 551 if conf.inputId is not None:
552 552 conf.object.setInput(self.configurations[conf.inputId].object)
553 553
554 554 def monitor(self):
555 555
556 556 t = Thread(target=self._monitor, args=(self.err_queue, self.ctx))
557 557 t.start()
558 558
559 559 def _monitor(self, queue, ctx):
560 560
561 561 import socket
562 562
563 563 procs = 0
564 564 err_msg = ''
565 565
566 566 while True:
567 567 msg = queue.get()
568 568 if '#_start_#' in msg:
569 569 procs += 1
570 570 elif '#_end_#' in msg:
571 571 procs -=1
572 572 else:
573 573 err_msg = msg
574 574
575 575 if procs == 0 or 'Traceback' in err_msg:
576 576 break
577 577 time.sleep(0.1)
578 578
579 579 if '|' in err_msg:
580 580 name, err = err_msg.split('|')
581 581 if 'SchainWarning' in err:
582 582 log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
583 583 elif 'SchainError' in err:
584 584 log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
585 585 else:
586 586 log.error(err, name)
587 587 else:
588 588 name, err = self.name, err_msg
589 589
590 590 time.sleep(1)
591 591
592 592 ctx.term()
593 593
594 594 message = ''.join(err)
595 595
596 596 if err_msg:
597 597 subject = 'SChain v%s: Error running %s\n' % (
598 598 schainpy.__version__, self.name)
599 599
600 600 subtitle = 'Hostname: %s\n' % socket.gethostbyname(
601 601 socket.gethostname())
602 602 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
603 603 subtitle += 'Configuration file: %s\n' % self.filename
604 604 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
605 605
606 606 readUnitConfObj = self.getReadUnit()
607 607 if readUnitConfObj:
608 608 subtitle += '\nInput parameters:\n'
609 609 subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters['path']
610 610 subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters['startDate']
611 611 subtitle += '[End date = %s]\n' % readUnitConfObj.parameters['endDate']
612 612 subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters['startTime']
613 613 subtitle += '[End time = %s]\n' % readUnitConfObj.parameters['endTime']
614 614
615 615 a = Alarm(
616 616 modes=self.alarm,
617 617 email=self.email,
618 618 message=message,
619 619 subject=subject,
620 620 subtitle=subtitle,
621 621 filename=self.filename
622 622 )
623 623
624 624 a.start()
625 625
626 626 def setFilename(self, filename):
627 627
628 628 self.filename = filename
629 629
630 630 def runProcs(self):
631 631
632 632 err = False
633 633 n = len(self.configurations)
634 634 flag_no_read = False
635 635 nProc_noRead = 0
636 while not err:
636 while 1:
637 637 #print("STAR")
638 638 n_proc = 0
639 639 for conf in self.getUnits():
640 640 #print("CONF: ",conf)
641 641 #print( n_proc, nProc_noRead,flag_no_read)
642 642 if flag_no_read:
643 643 if n_proc >= nProc_noRead:
644 644 ok = conf.run()
645 645 else:
646 646 #print("break")
647 647 #print(ok, n_proc, nProc_noRead, n, flag_no_read)
648 648 n_proc += 1
649 649 continue
650 650 else:
651 651 ok = conf.run()
652 652
653 653 n_proc += 1
654 654 #print(ok, n_proc, nProc_noRead, n, flag_no_read)
655 655
656 656 if ok == 'Error':
657 657 #self.removeProcUnit(conf.id) #remove proc Unit
658 658 n -= 1
659 659 continue
660 660
661 661 elif ok == 'no_Read' and (not flag_no_read):
662 662 nProc_noRead = n_proc - 1
663 663 flag_no_read = True
664 664 continue
665 665 elif ok == 'new_Read':
666 666 nProc_noRead = 0
667 667 flag_no_read = False
668 668 continue
669 669 elif not ok:
670 670 #print("not ok",ok)
671 671 break
672 672
673 673 if n == 0:
674 674 err = True
675 break
675 676
676 677 def run(self):
677 678
678 679 log.success('\nStarting Project {} [id={}]'.format(self.name, self.id), tag='')
679 680 self.started = True
680 681 self.start_time = time.time()
681 682 self.createObjects()
682 683 self.runProcs()
683 684 log.success('{} Done (Time: {:4.2f}s)'.format(
684 685 self.name,
685 686 time.time()-self.start_time), '')
Show file before | Show file after | Hide comments
@@ -1,1102 +1,1104
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Definition of diferent Data objects for different types of data
6 6
7 7 Here you will find the diferent data objects for the different types
8 8 of data, this data objects must be used as dataIn or dataOut objects in
9 9 processing units and operations. Currently the supported data objects are:
10 10 Voltage, Spectra, SpectraHeis, Fits, Correlation and Parameters
11 11 """
12 12
13 13 import copy
14 14 import numpy
15 15 import datetime
16 16 import json
17 17
18 18 import schainpy.admin
19 19 from schainpy.utils import log
20 20 from .jroheaderIO import SystemHeader, RadarControllerHeader
21 21 from schainpy.model.data import _noise
22 22
23 23
24 24 def getNumpyDtype(dataTypeCode):
25 25
26 26 if dataTypeCode == 0:
27 27 numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
28 28 elif dataTypeCode == 1:
29 29 numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
30 30 elif dataTypeCode == 2:
31 31 numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
32 32 elif dataTypeCode == 3:
33 33 numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
34 34 elif dataTypeCode == 4:
35 35 numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
36 36 elif dataTypeCode == 5:
37 37 numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
38 38 else:
39 39 raise ValueError('dataTypeCode was not defined')
40 40
41 41 return numpyDtype
42 42
43 43
44 44 def getDataTypeCode(numpyDtype):
45 45
46 46 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
47 47 datatype = 0
48 48 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
49 49 datatype = 1
50 50 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
51 51 datatype = 2
52 52 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
53 53 datatype = 3
54 54 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
55 55 datatype = 4
56 56 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
57 57 datatype = 5
58 58 else:
59 59 datatype = None
60 60
61 61 return datatype
62 62
63 63
64 64 def hildebrand_sekhon(data, navg):
65 65 """
66 66 This method is for the objective determination of the noise level in Doppler spectra. This
67 67 implementation technique is based on the fact that the standard deviation of the spectral
68 68 densities is equal to the mean spectral density for white Gaussian noise
69 69
70 70 Inputs:
71 71 Data : heights
72 72 navg : numbers of averages
73 73
74 74 Return:
75 75 mean : noise's level
76 76 """
77 77
78 78 sortdata = numpy.sort(data, axis=None)
79 79 '''
80 80 lenOfData = len(sortdata)
81 81 nums_min = lenOfData*0.5
82 82
83 83 if nums_min <= 5:
84 84
85 85 nums_min = 5
86 86
87 87 sump = 0.
88 88 sumq = 0.
89 89
90 90 j = 0
91 91 cont = 1
92 92
93 93 while((cont == 1)and(j < lenOfData)):
94 94
95 95 sump += sortdata[j]
96 96 sumq += sortdata[j]**2
97 97
98 98 if j > nums_min:
99 99 rtest = float(j)/(j-1) + 1.0/navg
100 100 if ((sumq*j) > (rtest*sump**2)):
101 101 j = j - 1
102 102 sump = sump - sortdata[j]
103 103 sumq = sumq - sortdata[j]**2
104 104 cont = 0
105 105
106 106 j += 1
107 107
108 108 lnoise = sump / j
109 109 return lnoise
110 110 '''
111 111 return _noise.hildebrand_sekhon(sortdata, navg)
112 112
113 113
114 114 class Beam:
115 115
116 116 def __init__(self):
117 117 self.codeList = []
118 118 self.azimuthList = []
119 119 self.zenithList = []
120 120
121 121
122 122
123 123 class GenericData(object):
124 124
125 125 flagNoData = True
126 126
127 127 def copy(self, inputObj=None):
128 128
129 129 if inputObj == None:
130 130 return copy.deepcopy(self)
131 131
132 132 for key in list(inputObj.__dict__.keys()):
133 133
134 134 attribute = inputObj.__dict__[key]
135 135
136 136 # If this attribute is a tuple or list
137 137 if type(inputObj.__dict__[key]) in (tuple, list):
138 138 self.__dict__[key] = attribute[:]
139 139 continue
140 140
141 141 # If this attribute is another object or instance
142 142 if hasattr(attribute, '__dict__'):
143 143 self.__dict__[key] = attribute.copy()
144 144 continue
145 145
146 146 self.__dict__[key] = inputObj.__dict__[key]
147 147
148 148 def deepcopy(self):
149 149
150 150 return copy.deepcopy(self)
151 151
152 152 def isEmpty(self):
153 153
154 154 return self.flagNoData
155 155
156 156 def isReady(self):
157 157
158 158 return not self.flagNoData
159 159
160 160
161 161 class JROData(GenericData):
162 162
163 163 useInputBuffer = False
164 164 buffer_empty = True
165 165
166 166 systemHeaderObj = SystemHeader()
167 167 radarControllerHeaderObj = RadarControllerHeader()
168 168 type = None
169 169 datatype = None # dtype but in string
170 170 nProfiles = None
171 171 heightList = None
172 172 channelList = None
173 173 flagDiscontinuousBlock = False
174 174 useLocalTime = False
175 175 utctime = None
176 176 timeZone = None
177 177 dstFlag = None
178 178 errorCount = None
179 179 blocksize = None
180 180 flagDecodeData = False # asumo q la data no esta decodificada
181 181 flagDeflipData = False # asumo q la data no esta sin flip
182 182 flagShiftFFT = False
183 183 nCohInt = None
184 184 windowOfFilter = 1
185 185 C = 3e8
186 186 frequency = 49.92e6
187 187 realtime = False
188 188 beacon_heiIndexList = None
189 189 last_block = None
190 190 blocknow = None
191 191 azimuth = None
192 192 zenith = None
193 193 beam = Beam()
194 194 profileIndex = None
195 195 error = None
196 196 data = None
197 197 nmodes = None
198 198 metadata_list = ['heightList', 'timeZone', 'type']
199 199 codeList = []
200 200 azimuthList = []
201 201 elevationList = []
202 last_noise = None
202 203
203 204 def __str__(self):
204 205
205 206 return '{} - {}'.format(self.type, self.datatime())
206 207
207 208 def getNoise(self):
208 209
209 210 raise NotImplementedError
210 211
211 212 @property
212 213 def nChannels(self):
213 214
214 215 return len(self.channelList)
215 216
216 217 @property
217 218 def channelIndexList(self):
218 219
219 220 return list(range(self.nChannels))
220 221
221 222 @property
222 223 def nHeights(self):
223 224
224 225 return len(self.heightList)
225 226
226 227 def getDeltaH(self):
227 228
228 229 return self.heightList[1] - self.heightList[0]
229 230
230 231 @property
231 232 def ltctime(self):
232 233
233 234 if self.useLocalTime:
234 235 return self.utctime - self.timeZone * 60
235 236
236 237 return self.utctime
237 238
238 239 @property
239 240 def datatime(self):
240 241
241 242 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
242 243 return datatimeValue
243 244
244 245 def getTimeRange(self):
245 246
246 247 datatime = []
247 248
248 249 datatime.append(self.ltctime)
249 250 datatime.append(self.ltctime + self.timeInterval + 1)
250 251
251 252 datatime = numpy.array(datatime)
252 253
253 254 return datatime
254 255
255 256 def getFmaxTimeResponse(self):
256 257
257 258 period = (10**-6) * self.getDeltaH() / (0.15)
258 259
259 260 PRF = 1. / (period * self.nCohInt)
260 261
261 262 fmax = PRF
262 263
263 264 return fmax
264 265
265 266 def getFmax(self):
266 267 PRF = 1. / (self.ippSeconds * self.nCohInt)
267 268
268 269 fmax = PRF
269 270 return fmax
270 271
271 272 def getVmax(self):
272 273
273 274 _lambda = self.C / self.frequency
274 275
275 276 vmax = self.getFmax() * _lambda / 2
276 277
277 278 return vmax
278 279
279 280 @property
280 281 def ippSeconds(self):
281 282 '''
282 283 '''
283 284 return self.radarControllerHeaderObj.ippSeconds
284 285
285 286 @ippSeconds.setter
286 287 def ippSeconds(self, ippSeconds):
287 288 '''
288 289 '''
289 290 self.radarControllerHeaderObj.ippSeconds = ippSeconds
290 291
291 292 @property
292 293 def code(self):
293 294 '''
294 295 '''
295 296 return self.radarControllerHeaderObj.code
296 297
297 298 @code.setter
298 299 def code(self, code):
299 300 '''
300 301 '''
301 302 self.radarControllerHeaderObj.code = code
302 303
303 304 @property
304 305 def nCode(self):
305 306 '''
306 307 '''
307 308 return self.radarControllerHeaderObj.nCode
308 309
309 310 @nCode.setter
310 311 def nCode(self, ncode):
311 312 '''
312 313 '''
313 314 self.radarControllerHeaderObj.nCode = ncode
314 315
315 316 @property
316 317 def nBaud(self):
317 318 '''
318 319 '''
319 320 return self.radarControllerHeaderObj.nBaud
320 321
321 322 @nBaud.setter
322 323 def nBaud(self, nbaud):
323 324 '''
324 325 '''
325 326 self.radarControllerHeaderObj.nBaud = nbaud
326 327
327 328 @property
328 329 def ipp(self):
329 330 '''
330 331 '''
331 332 return self.radarControllerHeaderObj.ipp
332 333
333 334 @ipp.setter
334 335 def ipp(self, ipp):
335 336 '''
336 337 '''
337 338 self.radarControllerHeaderObj.ipp = ipp
338 339
339 340 @property
340 341 def metadata(self):
341 342 '''
342 343 '''
343 344
344 345 return {attr: getattr(self, attr) for attr in self.metadata_list}
345 346
346 347
347 348 class Voltage(JROData):
348 349
349 350 dataPP_POW = None
350 351 dataPP_DOP = None
351 352 dataPP_WIDTH = None
352 353 dataPP_SNR = None
353 354
354 355 def __init__(self):
355 356 '''
356 357 Constructor
357 358 '''
358 359
359 360 self.useLocalTime = True
360 361 self.radarControllerHeaderObj = RadarControllerHeader()
361 362 self.systemHeaderObj = SystemHeader()
362 363 self.type = "Voltage"
363 364 self.data = None
364 365 self.nProfiles = None
365 366 self.heightList = None
366 367 self.channelList = None
367 368 self.flagNoData = True
368 369 self.flagDiscontinuousBlock = False
369 370 self.utctime = None
370 371 self.timeZone = 0
371 372 self.dstFlag = None
372 373 self.errorCount = None
373 374 self.nCohInt = None
374 375 self.blocksize = None
375 376 self.flagCohInt = False
376 377 self.flagDecodeData = False # asumo q la data no esta decodificada
377 378 self.flagDeflipData = False # asumo q la data no esta sin flip
378 379 self.flagShiftFFT = False
379 380 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
380 381 self.profileIndex = 0
381 382 self.metadata_list = ['type', 'heightList', 'timeZone', 'nProfiles', 'channelList', 'nCohInt',
382 383 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp']
383 384
384 385 def getNoisebyHildebrand(self, channel=None, ymin_index=None, ymax_index=None):
385 386 """
386 387 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
387 388
388 389 Return:
389 390 noiselevel
390 391 """
391 392
392 393 if channel != None:
393 394 data = self.data[channel,ymin_index:ymax_index]
394 395 nChannels = 1
395 396 else:
396 397 data = self.data[:,ymin_index:ymax_index]
397 398 nChannels = self.nChannels
398 399
399 400 noise = numpy.zeros(nChannels)
400 401 power = data * numpy.conjugate(data)
401 402
402 403 for thisChannel in range(nChannels):
403 404 if nChannels == 1:
404 405 daux = power[:].real
405 406 else:
406 407 daux = power[thisChannel, :].real
407 408 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
408 409
409 410 return noise
410 411
411 412 def getNoise(self, type=1, channel=None,ymin_index=None, ymax_index=None):
412 413
413 414 if type == 1:
414 415 noise = self.getNoisebyHildebrand(channel,ymin_index, ymax_index)
415 416
416 417 return noise
417 418
418 419 def getPower(self, channel=None):
419 420
420 421 if channel != None:
421 422 data = self.data[channel]
422 423 else:
423 424 data = self.data
424 425
425 426 power = data * numpy.conjugate(data)
426 427 powerdB = 10 * numpy.log10(power.real)
427 428 powerdB = numpy.squeeze(powerdB)
428 429
429 430 return powerdB
430 431
431 432 @property
432 433 def timeInterval(self):
433 434
434 435 return self.ippSeconds * self.nCohInt
435 436
436 437 noise = property(getNoise, "I'm the 'nHeights' property.")
437 438
438 439
439 440 class Spectra(JROData):
440 441
441 442 data_outlier = 0
442 443
443 444 def __init__(self):
444 445 '''
445 446 Constructor
446 447 '''
447 448
448 449 self.data_dc = None
449 450 self.data_spc = None
450 451 self.data_cspc = None
451 452 self.useLocalTime = True
452 453 self.radarControllerHeaderObj = RadarControllerHeader()
453 454 self.systemHeaderObj = SystemHeader()
454 455 self.type = "Spectra"
455 456 self.timeZone = 0
456 457 self.nProfiles = None
457 458 self.heightList = None
458 459 self.channelList = None
459 460 self.pairsList = None
460 461 self.flagNoData = True
461 462 self.flagDiscontinuousBlock = False
462 463 self.utctime = None
463 464 self.nCohInt = None
464 465 self.nIncohInt = None
465 466 self.blocksize = None
466 467 self.nFFTPoints = None
467 468 self.wavelength = None
468 469 self.flagDecodeData = False # asumo q la data no esta decodificada
469 470 self.flagDeflipData = False # asumo q la data no esta sin flip
470 471 self.flagShiftFFT = False
471 472 self.ippFactor = 1
472 473 self.beacon_heiIndexList = []
473 474 self.noise_estimation = None
474 475 self.codeList = []
475 476 self.azimuthList = []
476 477 self.elevationList = []
477 478 self.metadata_list = ['type', 'heightList', 'timeZone', 'pairsList', 'channelList', 'nCohInt',
478 479 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp','nIncohInt', 'nFFTPoints', 'nProfiles']
479 480
480 481
481 482 self.max_nIncohInt = 1
482 483
483 484
484 485 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
485 486 """
486 487 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
487 488
488 489 Return:
489 490 noiselevel
490 491 """
491 492 # if hasattr(self.nIncohInt, "__len__"): #nIncohInt is a matrix
492 493 #
493 494 # heis = self.data_spc.shape[2]
494 495 #
495 496 # noise = numpy.zeros((self.nChannels, heis))
496 497 # for hei in range(heis):
497 498 # for channel in range(self.nChannels):
498 499 # daux = self.data_spc[channel, xmin_index:xmax_index, hei]
499 500 #
500 501 # noise[channel,hei] = hildebrand_sekhon(daux, self.nIncohInt[channel,hei])
501 502 #
502 503 # else:
503 504 # noise = numpy.zeros(self.nChannels)
504 505 # for channel in range(self.nChannels):
505 506 # daux = self.data_spc[channel,xmin_index:xmax_index, ymin_index:ymax_index]
506 507 #
507 508 # noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
508 509 noise = numpy.zeros(self.nChannels)
509 510 for channel in range(self.nChannels):
510 511 daux = self.data_spc[channel,xmin_index:xmax_index, ymin_index:ymax_index]
511 512
512 513 noise[channel] = hildebrand_sekhon(daux, self.max_nIncohInt)
513 514
514 515 return noise
515 516
516 517 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
517 518
518 519 if self.noise_estimation is not None:
519 520 # this was estimated by getNoise Operation defined in jroproc_spectra.py
520 521 return self.noise_estimation
521 522 else:
522 523 noise = self.getNoisebyHildebrand(
523 524 xmin_index, xmax_index, ymin_index, ymax_index)
524 525 return noise
525 526
526 527 def getFreqRangeTimeResponse(self, extrapoints=0):
527 528
528 529 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
529 530 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.) - deltafreq / 2
530 531
531 532 return freqrange
532 533
533 534 def getAcfRange(self, extrapoints=0):
534 535
535 536 deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
536 537 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
537 538
538 539 return freqrange
539 540
540 541 def getFreqRange(self, extrapoints=0):
541 542
542 543 deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
543 544 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
544 545
545 546 return freqrange
546 547
547 548 def getVelRange(self, extrapoints=0):
548 549
549 550 deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
550 551 velrange = deltav * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.)
551 552
552 553 if self.nmodes:
553 554 return velrange/self.nmodes
554 555 else:
555 556 return velrange
556 557
557 558 @property
558 559 def nPairs(self):
559 560
560 561 return len(self.pairsList)
561 562
562 563 @property
563 564 def pairsIndexList(self):
564 565
565 566 return list(range(self.nPairs))
566 567
567 568 @property
568 569 def normFactor(self):
569 570
570 571 pwcode = 1
571 572
572 573 if self.flagDecodeData:
573 574 pwcode = numpy.sum(self.code[0]**2)
574 575 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
575 576 normFactor = self.nProfiles * self.nIncohInt * self.nCohInt * pwcode * self.windowOfFilter
576 577
577 578
578 579 return normFactor
579 580
580 581 @property
581 582 def flag_cspc(self):
582 583
583 584 if self.data_cspc is None:
584 585 return True
585 586
586 587 return False
587 588
588 589 @property
589 590 def flag_dc(self):
590 591
591 592 if self.data_dc is None:
592 593 return True
593 594
594 595 return False
595 596
596 597 @property
597 598 def timeInterval(self):
598 599
599 600 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
600 601 if self.nmodes:
601 602 return self.nmodes*timeInterval
602 603 else:
603 604 return timeInterval
604 605
605 606 def getPower(self):
606 607
607 608 factor = self.normFactor
608 609 z = numpy.divide(self.data_spc,factor)
609 610 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
610 611 avg = numpy.average(z, axis=1)
611 612
612 613 return 10 * numpy.log10(avg)
613 614
614 615 def getCoherence(self, pairsList=None, phase=False):
615 616
616 617 z = []
617 618 if pairsList is None:
618 619 pairsIndexList = self.pairsIndexList
619 620 else:
620 621 pairsIndexList = []
621 622 for pair in pairsList:
622 623 if pair not in self.pairsList:
623 624 raise ValueError("Pair %s is not in dataOut.pairsList" % (
624 625 pair))
625 626 pairsIndexList.append(self.pairsList.index(pair))
626 627 for i in range(len(pairsIndexList)):
627 628 pair = self.pairsList[pairsIndexList[i]]
628 629 ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
629 630 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
630 631 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
631 632 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
632 633 if phase:
633 634 data = numpy.arctan2(avgcoherenceComplex.imag,
634 635 avgcoherenceComplex.real) * 180 / numpy.pi
635 636 else:
636 637 data = numpy.abs(avgcoherenceComplex)
637 638
638 639 z.append(data)
639 640
640 641 return numpy.array(z)
641 642
642 643 def setValue(self, value):
643 644
644 print("This property should not be initialized")
645 print("This property should not be initialized", value)
645 646
646 647 return
647 648
648 649 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
649 650
650 651
651 652 class SpectraHeis(Spectra):
652 653
653 654 def __init__(self):
654 655
655 656 self.radarControllerHeaderObj = RadarControllerHeader()
656 657 self.systemHeaderObj = SystemHeader()
657 658 self.type = "SpectraHeis"
658 659 self.nProfiles = None
659 660 self.heightList = None
660 661 self.channelList = None
661 662 self.flagNoData = True
662 663 self.flagDiscontinuousBlock = False
663 664 self.utctime = None
664 665 self.blocksize = None
665 666 self.profileIndex = 0
666 667 self.nCohInt = 1
667 668 self.nIncohInt = 1
668 669
669 670 @property
670 671 def normFactor(self):
671 672 pwcode = 1
672 673 if self.flagDecodeData:
673 674 pwcode = numpy.sum(self.code[0]**2)
674 675
675 676 normFactor = self.nIncohInt * self.nCohInt * pwcode
676 677
677 678 return normFactor
678 679
679 680 @property
680 681 def timeInterval(self):
681 682
682 683 return self.ippSeconds * self.nCohInt * self.nIncohInt
683 684
684 685
685 686 class Fits(JROData):
686 687
687 688 def __init__(self):
688 689
689 690 self.type = "Fits"
690 691 self.nProfiles = None
691 692 self.heightList = None
692 693 self.channelList = None
693 694 self.flagNoData = True
694 695 self.utctime = None
695 696 self.nCohInt = 1
696 697 self.nIncohInt = 1
697 698 self.useLocalTime = True
698 699 self.profileIndex = 0
699 700 self.timeZone = 0
700 701
701 702 def getTimeRange(self):
702 703
703 704 datatime = []
704 705
705 706 datatime.append(self.ltctime)
706 707 datatime.append(self.ltctime + self.timeInterval)
707 708
708 709 datatime = numpy.array(datatime)
709 710
710 711 return datatime
711 712
712 713 def getChannelIndexList(self):
713 714
714 715 return list(range(self.nChannels))
715 716
716 717 def getNoise(self, type=1):
717 718
718 719
719 720 if type == 1:
720 721 noise = self.getNoisebyHildebrand()
721 722
722 723 if type == 2:
723 724 noise = self.getNoisebySort()
724 725
725 726 if type == 3:
726 727 noise = self.getNoisebyWindow()
727 728
728 729 return noise
729 730
730 731 @property
731 732 def timeInterval(self):
732 733
733 734 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
734 735
735 736 return timeInterval
736 737
737 738 @property
738 739 def ippSeconds(self):
739 740 '''
740 741 '''
741 742 return self.ipp_sec
742 743
743 744 noise = property(getNoise, "I'm the 'nHeights' property.")
744 745
745 746
746 747 class Correlation(JROData):
747 748
748 749 def __init__(self):
749 750 '''
750 751 Constructor
751 752 '''
752 753 self.radarControllerHeaderObj = RadarControllerHeader()
753 754 self.systemHeaderObj = SystemHeader()
754 755 self.type = "Correlation"
755 756 self.data = None
756 757 self.dtype = None
757 758 self.nProfiles = None
758 759 self.heightList = None
759 760 self.channelList = None
760 761 self.flagNoData = True
761 762 self.flagDiscontinuousBlock = False
762 763 self.utctime = None
763 764 self.timeZone = 0
764 765 self.dstFlag = None
765 766 self.errorCount = None
766 767 self.blocksize = None
767 768 self.flagDecodeData = False # asumo q la data no esta decodificada
768 769 self.flagDeflipData = False # asumo q la data no esta sin flip
769 770 self.pairsList = None
770 771 self.nPoints = None
771 772
772 773 def getPairsList(self):
773 774
774 775 return self.pairsList
775 776
776 777 def getNoise(self, mode=2):
777 778
778 779 indR = numpy.where(self.lagR == 0)[0][0]
779 780 indT = numpy.where(self.lagT == 0)[0][0]
780 781
781 782 jspectra0 = self.data_corr[:, :, indR, :]
782 783 jspectra = copy.copy(jspectra0)
783 784
784 785 num_chan = jspectra.shape[0]
785 786 num_hei = jspectra.shape[2]
786 787
787 788 freq_dc = jspectra.shape[1] / 2
788 789 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
789 790
790 791 if ind_vel[0] < 0:
791 792 ind_vel[list(range(0, 1))] = ind_vel[list(
792 793 range(0, 1))] + self.num_prof
793 794
794 795 if mode == 1:
795 796 jspectra[:, freq_dc, :] = (
796 797 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
797 798
798 799 if mode == 2:
799 800
800 801 vel = numpy.array([-2, -1, 1, 2])
801 802 xx = numpy.zeros([4, 4])
802 803
803 804 for fil in range(4):
804 805 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
805 806
806 807 xx_inv = numpy.linalg.inv(xx)
807 808 xx_aux = xx_inv[0, :]
808 809
809 810 for ich in range(num_chan):
810 811 yy = jspectra[ich, ind_vel, :]
811 812 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
812 813
813 814 junkid = jspectra[ich, freq_dc, :] <= 0
814 815 cjunkid = sum(junkid)
815 816
816 817 if cjunkid.any():
817 818 jspectra[ich, freq_dc, junkid.nonzero()] = (
818 819 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
819 820
820 821 noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
821 822
822 823 return noise
823 824
824 825 @property
825 826 def timeInterval(self):
826 827
827 828 return self.ippSeconds * self.nCohInt * self.nProfiles
828 829
829 830 def splitFunctions(self):
830 831
831 832 pairsList = self.pairsList
832 833 ccf_pairs = []
833 834 acf_pairs = []
834 835 ccf_ind = []
835 836 acf_ind = []
836 837 for l in range(len(pairsList)):
837 838 chan0 = pairsList[l][0]
838 839 chan1 = pairsList[l][1]
839 840
840 841 # Obteniendo pares de Autocorrelacion
841 842 if chan0 == chan1:
842 843 acf_pairs.append(chan0)
843 844 acf_ind.append(l)
844 845 else:
845 846 ccf_pairs.append(pairsList[l])
846 847 ccf_ind.append(l)
847 848
848 849 data_acf = self.data_cf[acf_ind]
849 850 data_ccf = self.data_cf[ccf_ind]
850 851
851 852 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
852 853
853 854 @property
854 855 def normFactor(self):
855 856 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
856 857 acf_pairs = numpy.array(acf_pairs)
857 858 normFactor = numpy.zeros((self.nPairs, self.nHeights))
858 859
859 860 for p in range(self.nPairs):
860 861 pair = self.pairsList[p]
861 862
862 863 ch0 = pair[0]
863 864 ch1 = pair[1]
864 865
865 866 ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
866 867 ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
867 868 normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
868 869
869 870 return normFactor
870 871
871 872
872 873 class Parameters(Spectra):
873 874
874 875 groupList = None # List of Pairs, Groups, etc
875 876 data_param = None # Parameters obtained
876 877 data_pre = None # Data Pre Parametrization
877 878 data_SNR = None # Signal to Noise Ratio
879 data_outlier = None
878 880 abscissaList = None # Abscissa, can be velocities, lags or time
879 881 utctimeInit = None # Initial UTC time
880 882 paramInterval = None # Time interval to calculate Parameters in seconds
881 883 useLocalTime = True
882 884 # Fitting
883 885 data_error = None # Error of the estimation
884 886 constants = None
885 887 library = None
886 888 # Output signal
887 889 outputInterval = None # Time interval to calculate output signal in seconds
888 890 data_output = None # Out signal
889 891 nAvg = None
890 892 noise_estimation = None
891 893 GauSPC = None # Fit gaussian SPC
892
894 max_nIncohInt = 1
893 895 def __init__(self):
894 896 '''
895 897 Constructor
896 898 '''
897 899 self.radarControllerHeaderObj = RadarControllerHeader()
898 900 self.systemHeaderObj = SystemHeader()
899 901 self.type = "Parameters"
900 902 self.timeZone = 0
901 903
902 904 def getTimeRange1(self, interval):
903 905
904 906 datatime = []
905 907
906 908 if self.useLocalTime:
907 909 time1 = self.utctimeInit - self.timeZone * 60
908 910 else:
909 911 time1 = self.utctimeInit
910 912
911 913 datatime.append(time1)
912 914 datatime.append(time1 + interval)
913 915 datatime = numpy.array(datatime)
914 916
915 917 return datatime
916 918
917 919 @property
918 920 def timeInterval(self):
919 921
920 922 if hasattr(self, 'timeInterval1'):
921 923 return self.timeInterval1
922 924 else:
923 925 return self.paramInterval
924 926
925 927 def setValue(self, value):
926 928
927 929 print("This property should not be initialized")
928 930
929 931 return
930 932
931 933 def getNoise(self):
932 934
933 935 return self.spc_noise
934 936
935 937 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
936 938
937 939
938 940 class PlotterData(object):
939 941 '''
940 942 Object to hold data to be plotted
941 943 '''
942 944
943 945 MAXNUMX = 200
944 946 MAXNUMY = 200
945 947
946 948 def __init__(self, code, exp_code, localtime=True):
947 949
948 950 self.key = code
949 951 self.exp_code = exp_code
950 952 self.ready = False
951 953 self.flagNoData = False
952 954 self.localtime = localtime
953 955 self.data = {}
954 956 self.meta = {}
955 957 self.__heights = []
956 958
957 959 def __str__(self):
958 960 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
959 961 return 'Data[{}][{}]'.format(';'.join(dum), len(self.times))
960 962
961 963 def __len__(self):
962 964 return len(self.data)
963 965
964 966 def __getitem__(self, key):
965 967 if isinstance(key, int):
966 968 return self.data[self.times[key]]
967 969 elif isinstance(key, str):
968 970 ret = numpy.array([self.data[x][key] for x in self.times])
969 971 if ret.ndim > 1:
970 972 ret = numpy.swapaxes(ret, 0, 1)
971 973 return ret
972 974
973 975 def __contains__(self, key):
974 976 return key in self.data[self.min_time]
975 977
976 978 def setup(self):
977 979 '''
978 980 Configure object
979 981 '''
980 982 self.type = ''
981 983 self.ready = False
982 984 del self.data
983 985 self.data = {}
984 986 self.__heights = []
985 987 self.__all_heights = set()
986 988
987 989 def shape(self, key):
988 990 '''
989 991 Get the shape of the one-element data for the given key
990 992 '''
991 993
992 994 if len(self.data[self.min_time][key]):
993 995 return self.data[self.min_time][key].shape
994 996 return (0,)
995 997
996 998 def update(self, data, tm, meta={}):
997 999 '''
998 1000 Update data object with new dataOut
999 1001 '''
1000 1002
1001 1003 self.data[tm] = data
1002 1004
1003 1005 for key, value in meta.items():
1004 1006 setattr(self, key, value)
1005 1007
1006 1008 def normalize_heights(self):
1007 1009 '''
1008 1010 Ensure same-dimension of the data for different heighList
1009 1011 '''
1010 1012
1011 1013 H = numpy.array(list(self.__all_heights))
1012 1014 H.sort()
1013 1015 for key in self.data:
1014 1016 shape = self.shape(key)[:-1] + H.shape
1015 1017 for tm, obj in list(self.data[key].items()):
1016 1018 h = self.__heights[self.times.tolist().index(tm)]
1017 1019 if H.size == h.size:
1018 1020 continue
1019 1021 index = numpy.where(numpy.in1d(H, h))[0]
1020 1022 dummy = numpy.zeros(shape) + numpy.nan
1021 1023 if len(shape) == 2:
1022 1024 dummy[:, index] = obj
1023 1025 else:
1024 1026 dummy[index] = obj
1025 1027 self.data[key][tm] = dummy
1026 1028
1027 1029 self.__heights = [H for tm in self.times]
1028 1030
1029 1031 def jsonify(self, tm, plot_name, plot_type, decimate=False):
1030 1032 '''
1031 1033 Convert data to json
1032 1034 '''
1033 1035
1034 1036 meta = {}
1035 1037 meta['xrange'] = []
1036 1038 dy = int(len(self.yrange)/self.MAXNUMY) + 1
1037 1039 tmp = self.data[tm][self.key]
1038 1040 shape = tmp.shape
1039 1041 if len(shape) == 2:
1040 1042 data = self.roundFloats(self.data[tm][self.key][::, ::dy].tolist())
1041 1043 elif len(shape) == 3:
1042 1044 dx = int(self.data[tm][self.key].shape[1]/self.MAXNUMX) + 1
1043 1045 data = self.roundFloats(
1044 1046 self.data[tm][self.key][::, ::dx, ::dy].tolist())
1045 1047 meta['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
1046 1048 else:
1047 1049 data = self.roundFloats(self.data[tm][self.key].tolist())
1048 1050
1049 1051 ret = {
1050 1052 'plot': plot_name,
1051 1053 'code': self.exp_code,
1052 1054 'time': float(tm),
1053 1055 'data': data,
1054 1056 }
1055 1057 meta['type'] = plot_type
1056 1058 meta['interval'] = float(self.interval)
1057 1059 meta['localtime'] = self.localtime
1058 1060 meta['yrange'] = self.roundFloats(self.yrange[::dy].tolist())
1059 1061 meta.update(self.meta)
1060 1062 ret['metadata'] = meta
1061 1063 return json.dumps(ret)
1062 1064
1063 1065 @property
1064 1066 def times(self):
1065 1067 '''
1066 1068 Return the list of times of the current data
1067 1069 '''
1068 1070
1069 1071 ret = [t for t in self.data]
1070 1072 ret.sort()
1071 1073 return numpy.array(ret)
1072 1074
1073 1075 @property
1074 1076 def min_time(self):
1075 1077 '''
1076 1078 Return the minimun time value
1077 1079 '''
1078 1080
1079 1081 return self.times[0]
1080 1082
1081 1083 @property
1082 1084 def max_time(self):
1083 1085 '''
1084 1086 Return the maximun time value
1085 1087 '''
1086 1088
1087 1089 return self.times[-1]
1088 1090
1089 1091 # @property
1090 1092 # def heights(self):
1091 1093 # '''
1092 1094 # Return the list of heights of the current data
1093 1095 # '''
1094 1096
1095 1097 # return numpy.array(self.__heights[-1])
1096 1098
1097 1099 @staticmethod
1098 1100 def roundFloats(obj):
1099 1101 if isinstance(obj, list):
1100 1102 return list(map(PlotterData.roundFloats, obj))
1101 1103 elif isinstance(obj, float):
1102 1104 return round(obj, 2)
Show file before | Show file after | Hide comments
@@ -1,705 +1,705
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Base class to create plot operations
6 6
7 7 """
8 8
9 9 import os
10 10 import sys
11 11 import zmq
12 12 import time
13 13 import numpy
14 14 import datetime
15 15 from collections import deque
16 16 from functools import wraps
17 17 from threading import Thread
18 18 import matplotlib
19 19
20 20 if 'BACKEND' in os.environ:
21 21 matplotlib.use(os.environ['BACKEND'])
22 22 elif 'linux' in sys.platform:
23 23 matplotlib.use("TkAgg")
24 24 elif 'darwin' in sys.platform:
25 25 matplotlib.use('MacOSX')
26 26 else:
27 27 from schainpy.utils import log
28 28 log.warning('Using default Backend="Agg"', 'INFO')
29 29 matplotlib.use('Agg')
30 30
31 31 import matplotlib.pyplot as plt
32 32 from matplotlib.patches import Polygon
33 33 from mpl_toolkits.axes_grid1 import make_axes_locatable
34 34 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
35 35
36 36 from schainpy.model.data.jrodata import PlotterData
37 37 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
38 38 from schainpy.utils import log
39 39
40 40 jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
41 41 blu_values = matplotlib.pyplot.get_cmap(
42 42 'seismic_r', 20)(numpy.arange(20))[10:15]
43 43 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
44 44 'jro', numpy.vstack((blu_values, jet_values)))
45 45 matplotlib.pyplot.register_cmap(cmap=ncmap)
46 46
47 47 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis',
48 48 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
49 49
50 50 EARTH_RADIUS = 6.3710e3
51 51
52 52 def ll2xy(lat1, lon1, lat2, lon2):
53 53
54 54 p = 0.017453292519943295
55 55 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
56 56 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
57 57 r = 12742 * numpy.arcsin(numpy.sqrt(a))
58 58 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
59 59 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
60 60 theta = -theta + numpy.pi/2
61 61 return r*numpy.cos(theta), r*numpy.sin(theta)
62 62
63 63
64 64 def km2deg(km):
65 65 '''
66 66 Convert distance in km to degrees
67 67 '''
68 68
69 69 return numpy.rad2deg(km/EARTH_RADIUS)
70 70
71 71
72 72 def figpause(interval):
73 73 backend = plt.rcParams['backend']
74 74 if backend in matplotlib.rcsetup.interactive_bk:
75 75 figManager = matplotlib._pylab_helpers.Gcf.get_active()
76 76 if figManager is not None:
77 77 canvas = figManager.canvas
78 78 if canvas.figure.stale:
79 79 canvas.draw()
80 80 try:
81 81 canvas.start_event_loop(interval)
82 82 except:
83 83 pass
84 84 return
85 85
86 86 def popup(message):
87 87 '''
88 88 '''
89 89
90 90 fig = plt.figure(figsize=(12, 8), facecolor='r')
91 91 text = '\n'.join([s.strip() for s in message.split(':')])
92 92 fig.text(0.01, 0.5, text, ha='left', va='center',
93 93 size='20', weight='heavy', color='w')
94 94 fig.show()
95 95 figpause(1000)
96 96
97 97
98 98 class Throttle(object):
99 99 '''
100 100 Decorator that prevents a function from being called more than once every
101 101 time period.
102 102 To create a function that cannot be called more than once a minute, but
103 103 will sleep until it can be called:
104 104 @Throttle(minutes=1)
105 105 def foo():
106 106 pass
107 107
108 108 for i in range(10):
109 109 foo()
110 110 print "This function has run %s times." % i
111 111 '''
112 112
113 113 def __init__(self, seconds=0, minutes=0, hours=0):
114 114 self.throttle_period = datetime.timedelta(
115 115 seconds=seconds, minutes=minutes, hours=hours
116 116 )
117 117
118 118 self.time_of_last_call = datetime.datetime.min
119 119
120 120 def __call__(self, fn):
121 121 @wraps(fn)
122 122 def wrapper(*args, **kwargs):
123 123 coerce = kwargs.pop('coerce', None)
124 124 if coerce:
125 125 self.time_of_last_call = datetime.datetime.now()
126 126 return fn(*args, **kwargs)
127 127 else:
128 128 now = datetime.datetime.now()
129 129 time_since_last_call = now - self.time_of_last_call
130 130 time_left = self.throttle_period - time_since_last_call
131 131
132 132 if time_left > datetime.timedelta(seconds=0):
133 133 return
134 134
135 135 self.time_of_last_call = datetime.datetime.now()
136 136 return fn(*args, **kwargs)
137 137
138 138 return wrapper
139 139
140 140 def apply_throttle(value):
141 141
142 142 @Throttle(seconds=value)
143 143 def fnThrottled(fn):
144 144 fn()
145 145
146 146 return fnThrottled
147 147
148 148
149 149 @MPDecorator
150 150 class Plot(Operation):
151 151 """Base class for Schain plotting operations
152 152
153 153 This class should never be use directtly you must subclass a new operation,
154 154 children classes must be defined as follow:
155 155
156 156 ExamplePlot(Plot):
157 157
158 158 CODE = 'code'
159 159 colormap = 'jet'
160 160 plot_type = 'pcolor' # options are ('pcolor', 'pcolorbuffer', 'scatter', 'scatterbuffer')
161 161
162 162 def setup(self):
163 163 pass
164 164
165 165 def plot(self):
166 166 pass
167 167
168 168 """
169 169
170 170 CODE = 'Figure'
171 171 colormap = 'jet'
172 172 bgcolor = 'white'
173 173 buffering = True
174 174 __missing = 1E30
175 175
176 176 __attrs__ = ['show', 'save', 'ymin', 'ymax', 'zmin', 'zmax', 'title',
177 177 'showprofile']
178 178
179 179 def __init__(self):
180 180
181 181 Operation.__init__(self)
182 182 self.isConfig = False
183 183 self.isPlotConfig = False
184 184 self.save_time = 0
185 185 self.sender_time = 0
186 186 self.data = None
187 187 self.firsttime = True
188 188 self.sender_queue = deque(maxlen=10)
189 189 self.plots_adjust = {'left': 0.125, 'right': 0.9, 'bottom': 0.15, 'top': 0.9, 'wspace': 0.2, 'hspace': 0.2}
190 190
191 191 def __fmtTime(self, x, pos):
192 192 '''
193 193 '''
194 194 if self.t_units == "h_m":
195 195 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
196 196 if self.t_units == "h":
197 197 return '{}'.format(self.getDateTime(x).strftime('%H'))
198 198
199 199 def __setup(self, **kwargs):
200 200 '''
201 201 Initialize variables
202 202 '''
203 203
204 204 self.figures = []
205 205 self.axes = []
206 206 self.cb_axes = []
207 207 self.pf_axes = []
208 208 self.localtime = kwargs.pop('localtime', True)
209 209 self.show = kwargs.get('show', True)
210 210 self.save = kwargs.get('save', False)
211 211 self.save_period = kwargs.get('save_period', 0)
212 212 self.colormap = kwargs.get('colormap', self.colormap)
213 213 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
214 214 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
215 215 self.colormaps = kwargs.get('colormaps', None)
216 216 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
217 217 self.showprofile = kwargs.get('showprofile', False)
218 218 self.title = kwargs.get('wintitle', self.CODE.upper())
219 219 self.cb_label = kwargs.get('cb_label', None)
220 220 self.cb_labels = kwargs.get('cb_labels', None)
221 221 self.labels = kwargs.get('labels', None)
222 222 self.xaxis = kwargs.get('xaxis', 'frequency')
223 223 self.zmin = kwargs.get('zmin', None)
224 224 self.zmax = kwargs.get('zmax', None)
225 225 self.zlimits = kwargs.get('zlimits', None)
226 226 self.xmin = kwargs.get('xmin', None)
227 227 self.xmax = kwargs.get('xmax', None)
228 228 self.xrange = kwargs.get('xrange', 12)
229 229 self.xscale = kwargs.get('xscale', None)
230 230 self.ymin = kwargs.get('ymin', None)
231 231 self.ymax = kwargs.get('ymax', None)
232 232 self.yscale = kwargs.get('yscale', None)
233 233 self.xlabel = kwargs.get('xlabel', None)
234 234 self.attr_time = kwargs.get('attr_time', 'utctime')
235 235 self.attr_data = kwargs.get('attr_data', 'data_param')
236 236 self.decimation = kwargs.get('decimation', None)
237 237 self.oneFigure = kwargs.get('oneFigure', True)
238 238 self.width = kwargs.get('width', None)
239 239 self.height = kwargs.get('height', None)
240 240 self.colorbar = kwargs.get('colorbar', True)
241 241 self.factors = kwargs.get('factors', range(18))
242 242 self.channels = kwargs.get('channels', None)
243 243 self.titles = kwargs.get('titles', [])
244 244 self.polar = False
245 245 self.type = kwargs.get('type', 'iq')
246 246 self.grid = kwargs.get('grid', False)
247 247 self.pause = kwargs.get('pause', False)
248 248 self.save_code = kwargs.get('save_code', self.CODE)
249 249 self.throttle = kwargs.get('throttle', 0)
250 250 self.exp_code = kwargs.get('exp_code', None)
251 251 self.server = kwargs.get('server', False)
252 252 self.sender_period = kwargs.get('sender_period', 60)
253 253 self.tag = kwargs.get('tag', '')
254 254 self.height_index = kwargs.get('height_index', [])
255 255 self.__throttle_plot = apply_throttle(self.throttle)
256 256 code = self.attr_data if self.attr_data else self.CODE
257 257 self.data = PlotterData(self.CODE, self.exp_code, self.localtime)
258 258 self.tmin = kwargs.get('tmin', None)
259 259 self.t_units = kwargs.get('t_units', "h_m")
260 260 self.selectedHeightsList = kwargs.get('selectedHeightsList', [])
261 261 if isinstance(self.selectedHeightsList, int):
262 262 self.selectedHeightsList = [self.selectedHeightsList]
263 263
264 264 if self.server:
265 265 if not self.server.startswith('tcp://'):
266 266 self.server = 'tcp://{}'.format(self.server)
267 267 log.success(
268 268 'Sending to server: {}'.format(self.server),
269 269 self.name
270 270 )
271 271
272 272 if isinstance(self.attr_data, str):
273 273 self.attr_data = [self.attr_data]
274 274
275 275 def __setup_plot(self):
276 276 '''
277 277 Common setup for all figures, here figures and axes are created
278 278 '''
279 279
280 280 self.setup()
281 281
282 282 self.time_label = 'LT' if self.localtime else 'UTC'
283 283
284 284 if self.width is None:
285 285 self.width = 8
286 286
287 287 self.figures = []
288 288 self.axes = []
289 289 self.cb_axes = []
290 290 self.pf_axes = []
291 291 self.cmaps = []
292 292
293 293 size = '15%' if self.ncols == 1 else '30%'
294 294 pad = '4%' if self.ncols == 1 else '8%'
295 295
296 296 if self.oneFigure:
297 297 if self.height is None:
298 298 self.height = 1.4 * self.nrows + 1
299 299 fig = plt.figure(figsize=(self.width, self.height),
300 300 edgecolor='k',
301 301 facecolor='w')
302 302 self.figures.append(fig)
303 303 for n in range(self.nplots):
304 304 ax = fig.add_subplot(self.nrows, self.ncols,
305 305 n + 1, polar=self.polar)
306 306 ax.tick_params(labelsize=8)
307 307 ax.firsttime = True
308 308 ax.index = 0
309 309 ax.press = None
310 310 self.axes.append(ax)
311 311 if self.showprofile:
312 312 cax = self.__add_axes(ax, size=size, pad=pad)
313 313 cax.tick_params(labelsize=8)
314 314 self.pf_axes.append(cax)
315 315 else:
316 316 if self.height is None:
317 317 self.height = 3
318 318 for n in range(self.nplots):
319 319 fig = plt.figure(figsize=(self.width, self.height),
320 320 edgecolor='k',
321 321 facecolor='w')
322 322 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
323 323 ax.tick_params(labelsize=8)
324 324 ax.firsttime = True
325 325 ax.index = 0
326 326 ax.press = None
327 327 self.figures.append(fig)
328 328 self.axes.append(ax)
329 329 if self.showprofile:
330 330 cax = self.__add_axes(ax, size=size, pad=pad)
331 331 cax.tick_params(labelsize=8)
332 332 self.pf_axes.append(cax)
333 333
334 334 for n in range(self.nrows):
335 335 if self.colormaps is not None:
336 336 cmap = plt.get_cmap(self.colormaps[n])
337 337 else:
338 338 cmap = plt.get_cmap(self.colormap)
339 339 cmap.set_bad(self.bgcolor, 1.)
340 340 self.cmaps.append(cmap)
341 341
342 342 def __add_axes(self, ax, size='30%', pad='8%'):
343 343 '''
344 344 Add new axes to the given figure
345 345 '''
346 346 divider = make_axes_locatable(ax)
347 347 nax = divider.new_horizontal(size=size, pad=pad)
348 348 ax.figure.add_axes(nax)
349 349 return nax
350 350
351 351 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
352 352 '''
353 353 Create a masked array for missing data
354 354 '''
355 355 if x_buffer.shape[0] < 2:
356 356 return x_buffer, y_buffer, z_buffer
357 357
358 358 deltas = x_buffer[1:] - x_buffer[0:-1]
359 359 x_median = numpy.median(deltas)
360 360
361 361 index = numpy.where(deltas > 5 * x_median)
362 362
363 363 if len(index[0]) != 0:
364 364 z_buffer[::, index[0], ::] = self.__missing
365 365 z_buffer = numpy.ma.masked_inside(z_buffer,
366 366 0.99 * self.__missing,
367 367 1.01 * self.__missing)
368 368
369 369 return x_buffer, y_buffer, z_buffer
370 370
371 371 def decimate(self):
372 372
373 373 # dx = int(len(self.x)/self.__MAXNUMX) + 1
374 374 dy = int(len(self.y) / self.decimation) + 1
375 375
376 376 # x = self.x[::dx]
377 377 x = self.x
378 378 y = self.y[::dy]
379 379 z = self.z[::, ::, ::dy]
380 380
381 381 return x, y, z
382 382
383 383 def format(self):
384 384 '''
385 385 Set min and max values, labels, ticks and titles
386 386 '''
387 387
388 388 for n, ax in enumerate(self.axes):
389 389 if ax.firsttime:
390 390 if self.xaxis != 'time':
391 391 xmin = self.xmin
392 392 xmax = self.xmax
393 393 else:
394 394 xmin = self.tmin
395 395 xmax = self.tmin + self.xrange*60*60
396 396 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
397 397 if self.t_units == "h_m":
398 398 ax.xaxis.set_major_locator(LinearLocator(9))
399 399 if self.t_units == "h":
400 400 ax.xaxis.set_major_locator(LinearLocator(int((xmax-xmin)/3600)+1))
401 401 ymin = self.ymin if self.ymin is not None else numpy.nanmin(self.y[numpy.isfinite(self.y)])
402 402 ymax = self.ymax if self.ymax is not None else numpy.nanmax(self.y[numpy.isfinite(self.y)])
403 403 ax.set_facecolor(self.bgcolor)
404 404 if self.xscale:
405 405 ax.xaxis.set_major_formatter(FuncFormatter(
406 406 lambda x, pos: '{0:g}'.format(x*self.xscale)))
407 407 if self.yscale:
408 408 ax.yaxis.set_major_formatter(FuncFormatter(
409 409 lambda x, pos: '{0:g}'.format(x*self.yscale)))
410 410 if self.xlabel is not None:
411 411 ax.set_xlabel(self.xlabel)
412 412 if self.ylabel is not None:
413 413 ax.set_ylabel(self.ylabel)
414 414 if self.showprofile:
415 415 self.pf_axes[n].set_ylim(ymin, ymax)
416 416 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
417 417 self.pf_axes[n].set_xlabel('dB')
418 418 self.pf_axes[n].grid(b=True, axis='x')
419 419 [tick.set_visible(False)
420 420 for tick in self.pf_axes[n].get_yticklabels()]
421 421 if self.colorbar:
422 422 ax.cbar = plt.colorbar(
423 423 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
424 424 ax.cbar.ax.tick_params(labelsize=8)
425 425 ax.cbar.ax.press = None
426 426 if self.cb_label:
427 427 ax.cbar.set_label(self.cb_label, size=8)
428 428 elif self.cb_labels:
429 429 ax.cbar.set_label(self.cb_labels[n], size=8)
430 430 else:
431 431 ax.cbar = None
432 432 ax.set_xlim(xmin, xmax)
433 433 ax.set_ylim(ymin, ymax)
434 434 ax.firsttime = False
435 435 if self.grid:
436 436 ax.grid(True)
437 437
438 438 if not self.polar:
439 439 ax.set_title('{} {} {}'.format(
440 440 self.titles[n],
441 441 self.getDateTime(self.data.max_time).strftime(
442 442 '%Y-%m-%d %H:%M:%S'),
443 443 self.time_label),
444 444 size=8)
445 445 else:
446 446
447 447 ax.set_title('{}'.format(self.titles[n]), size=8)
448 448 ax.set_ylim(0, 90)
449 449 ax.set_yticks(numpy.arange(0, 90, 20))
450 450 ax.yaxis.labelpad = 40
451 451
452 452 if self.firsttime:
453 453 for n, fig in enumerate(self.figures):
454 454 fig.subplots_adjust(**self.plots_adjust)
455 455 self.firsttime = False
456 456
457 457 def clear_figures(self):
458 458 '''
459 459 Reset axes for redraw plots
460 460 '''
461 461
462 462 for ax in self.axes+self.pf_axes+self.cb_axes:
463 463 ax.clear()
464 464 ax.firsttime = True
465 465 if hasattr(ax, 'cbar') and ax.cbar:
466 466 ax.cbar.remove()
467 467
468 468 def __plot(self):
469 469 '''
470 470 Main function to plot, format and save figures
471 471 '''
472 472
473 473 self.plot()
474 474 self.format()
475 475
476 476 for n, fig in enumerate(self.figures):
477 477 if self.nrows == 0 or self.nplots == 0:
478 478 log.warning('No data', self.name)
479 479 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
480 480 fig.canvas.manager.set_window_title(self.CODE)
481 481 continue
482 482
483 483 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
484 484 self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
485 485
486 486 fig.canvas.draw()
487 487 if self.show:
488 488 fig.show()
489 489 figpause(0.01)
490 490
491 491 if self.save:
492 492 self.save_figure(n)
493 493
494 494 if self.server:
495 495 self.send_to_server()
496 496
497 497 def __update(self, dataOut, timestamp):
498 498 '''
499 499 '''
500 500
501 501 metadata = {
502 502 'yrange': dataOut.heightList,
503 503 'interval': dataOut.timeInterval,
504 504 'channels': dataOut.channelList
505 505 }
506 506 data, meta = self.update(dataOut)
507 507 metadata.update(meta)
508 508 self.data.update(data, timestamp, metadata)
509 509
510 510 def save_figure(self, n):
511 511 '''
512 512 '''
513 513
514 514 if (self.data.max_time - self.save_time) <= self.save_period:
515 515 return
516 516
517 517 self.save_time = self.data.max_time
518 518
519 519 fig = self.figures[n]
520 520
521 521 if self.throttle == 0:
522 522 figname = os.path.join(
523 523 self.save,
524 524 self.save_code,
525 525 '{}_{}.png'.format(
526 526 self.save_code,
527 527 self.getDateTime(self.data.max_time).strftime(
528 528 '%Y%m%d_%H%M%S'
529 529 ),
530 530 )
531 531 )
532 532 log.log('Saving figure: {}'.format(figname), self.name)
533 533 if not os.path.isdir(os.path.dirname(figname)):
534 534 os.makedirs(os.path.dirname(figname))
535 535 fig.savefig(figname)
536 536
537 537 figname = os.path.join(
538 538 self.save,
539 539 '{}_{}.png'.format(
540 540 self.save_code,
541 541 self.getDateTime(self.data.min_time).strftime(
542 542 '%Y%m%d'
543 543 ),
544 544 )
545 545 )
546 546
547 547 log.log('Saving figure: {}'.format(figname), self.name)
548 548 if not os.path.isdir(os.path.dirname(figname)):
549 549 os.makedirs(os.path.dirname(figname))
550 550 fig.savefig(figname)
551 551
552 552 def send_to_server(self):
553 553 '''
554 554 '''
555 555
556 556 if self.exp_code == None:
557 557 log.warning('Missing `exp_code` skipping sending to server...')
558 558
559 559 last_time = self.data.max_time
560 560 interval = last_time - self.sender_time
561 561 if interval < self.sender_period:
562 562 return
563 563
564 564 self.sender_time = last_time
565 565
566 566 attrs = ['titles', 'zmin', 'zmax', 'tag', 'ymin', 'ymax']
567 567 for attr in attrs:
568 568 value = getattr(self, attr)
569 569 if value:
570 570 if isinstance(value, (numpy.float32, numpy.float64)):
571 571 value = round(float(value), 2)
572 572 self.data.meta[attr] = value
573 573 if self.colormap == 'jet':
574 574 self.data.meta['colormap'] = 'Jet'
575 575 elif 'RdBu' in self.colormap:
576 576 self.data.meta['colormap'] = 'RdBu'
577 577 else:
578 578 self.data.meta['colormap'] = 'Viridis'
579 579 self.data.meta['interval'] = int(interval)
580 580
581 581 self.sender_queue.append(last_time)
582 582
583 while True:
583 while 1:
584 584 try:
585 585 tm = self.sender_queue.popleft()
586 586 except IndexError:
587 587 break
588 588 msg = self.data.jsonify(tm, self.save_code, self.plot_type)
589 589 self.socket.send_string(msg)
590 590 socks = dict(self.poll.poll(2000))
591 591 if socks.get(self.socket) == zmq.POLLIN:
592 592 reply = self.socket.recv_string()
593 593 if reply == 'ok':
594 594 log.log("Response from server ok", self.name)
595 595 time.sleep(0.1)
596 596 continue
597 597 else:
598 598 log.warning(
599 599 "Malformed reply from server: {}".format(reply), self.name)
600 600 else:
601 601 log.warning(
602 602 "No response from server, retrying...", self.name)
603 603 self.sender_queue.appendleft(tm)
604 604 self.socket.setsockopt(zmq.LINGER, 0)
605 605 self.socket.close()
606 606 self.poll.unregister(self.socket)
607 607 self.socket = self.context.socket(zmq.REQ)
608 608 self.socket.connect(self.server)
609 609 self.poll.register(self.socket, zmq.POLLIN)
610 610 break
611 611
612 612 def setup(self):
613 613 '''
614 614 This method should be implemented in the child class, the following
615 615 attributes should be set:
616 616
617 617 self.nrows: number of rows
618 618 self.ncols: number of cols
619 619 self.nplots: number of plots (channels or pairs)
620 620 self.ylabel: label for Y axes
621 621 self.titles: list of axes title
622 622
623 623 '''
624 624 raise NotImplementedError
625 625
626 626 def plot(self):
627 627 '''
628 628 Must be defined in the child class, the actual plotting method
629 629 '''
630 630 raise NotImplementedError
631 631
632 632 def update(self, dataOut):
633 633 '''
634 634 Must be defined in the child class, update self.data with new data
635 635 '''
636 636
637 637 data = {
638 638 self.CODE: getattr(dataOut, 'data_{}'.format(self.CODE))
639 639 }
640 640 meta = {}
641 641
642 642 return data, meta
643 643
644 644 def run(self, dataOut, **kwargs):
645 645 '''
646 646 Main plotting routine
647 647 '''
648 648 if self.isConfig is False:
649 649 self.__setup(**kwargs)
650 650
651 651 if self.localtime:
652 652 self.getDateTime = datetime.datetime.fromtimestamp
653 653 else:
654 654 self.getDateTime = datetime.datetime.utcfromtimestamp
655 655
656 656 self.data.setup()
657 657 self.isConfig = True
658 658 if self.server:
659 659 self.context = zmq.Context()
660 660 self.socket = self.context.socket(zmq.REQ)
661 661 self.socket.connect(self.server)
662 662 self.poll = zmq.Poller()
663 663 self.poll.register(self.socket, zmq.POLLIN)
664 664
665 665 tm = getattr(dataOut, self.attr_time)
666 666
667 667 if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60:
668 668 self.save_time = tm
669 669 self.__plot()
670 670 self.tmin += self.xrange*60*60
671 671 self.data.setup()
672 672 self.clear_figures()
673 673
674 674 self.__update(dataOut, tm)
675 675
676 676 if self.isPlotConfig is False:
677 677 self.__setup_plot()
678 678 self.isPlotConfig = True
679 679 if self.xaxis == 'time':
680 680 dt = self.getDateTime(tm)
681 681 if self.xmin is None:
682 682 self.tmin = tm
683 683 self.xmin = dt.hour
684 684 minutes = (self.xmin-int(self.xmin)) * 60
685 685 seconds = (minutes - int(minutes)) * 60
686 686 self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) -
687 687 datetime.datetime(1970, 1, 1)).total_seconds()
688 688 if self.localtime:
689 689 self.tmin += time.timezone
690 690
691 691 if self.xmin is not None and self.xmax is not None:
692 692 self.xrange = self.xmax - self.xmin
693 693
694 694 if self.throttle == 0:
695 695 self.__plot()
696 696 else:
697 697 self.__throttle_plot(self.__plot)#, coerce=coerce)
698 698
699 699 def close(self):
700 700
701 701 if self.data and not self.data.flagNoData:
702 702 self.save_time = 0
703 703 self.__plot()
704 704 if self.data and not self.data.flagNoData and self.pause:
705 705 figpause(10)
Show file before | Show file after | Hide comments
@@ -1,362 +1,362
1 1 import os
2 2 import datetime
3 3 import numpy
4 4
5 5 from schainpy.model.graphics.jroplot_base import Plot, plt
6 6 from schainpy.model.graphics.jroplot_spectra import SpectraPlot, RTIPlot, CoherencePlot
7 7 from schainpy.utils import log
8 8
9 9 EARTH_RADIUS = 6.3710e3
10 10
11 11
12 12 def ll2xy(lat1, lon1, lat2, lon2):
13 13
14 14 p = 0.017453292519943295
15 15 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
16 16 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
17 17 r = 12742 * numpy.arcsin(numpy.sqrt(a))
18 18 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
19 19 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
20 20 theta = -theta + numpy.pi/2
21 21 return r*numpy.cos(theta), r*numpy.sin(theta)
22 22
23 23
24 24 def km2deg(km):
25 25 '''
26 26 Convert distance in km to degrees
27 27 '''
28 28
29 29 return numpy.rad2deg(km/EARTH_RADIUS)
30 30
31 31
32 32
33 33 class SpectralMomentsPlot(SpectraPlot):
34 34 '''
35 35 Plot for Spectral Moments
36 36 '''
37 37 CODE = 'spc_moments'
38 38 colormap = 'jet'
39 39 plot_type = 'pcolor'
40 40
41 41
42 42 class SnrPlot(RTIPlot):
43 43 '''
44 44 Plot for SNR Data
45 45 '''
46 46
47 47 CODE = 'snr'
48 48 colormap = 'jet'
49 49
50 50 def update(self, dataOut):
51 51 if len(self.channelList) == 0:
52 52 self.update_list(dataOut)
53 53
54 54 meta = {}
55 55 data = {
56 'snr': 10 * numpy.log10(dataOut.data_snr)
56 'snr': 10 * numpy.log10(dataOut.data_snr * dataOut.nIncohInt/ dataOut.max_nIncohInt)
57 57 }
58 58 #print(data['snr'])
59 59 return data, meta
60 60
61 61 class DopplerPlot(RTIPlot):
62 62 '''
63 63 Plot for DOPPLER Data (1st moment)
64 64 '''
65 65
66 66 CODE = 'dop'
67 67 colormap = 'jet'
68 68
69 69 def update(self, dataOut):
70 70 self.update_list(dataOut)
71 71 data = {
72 72 'dop': 10*numpy.log10(dataOut.data_dop)
73 73 }
74 74
75 75 return data, {}
76 76
77 77 class PowerPlot(RTIPlot):
78 78 '''
79 79 Plot for Power Data (0 moment)
80 80 '''
81 81
82 82 CODE = 'pow'
83 83 colormap = 'jet'
84 84
85 85 def update(self, dataOut):
86 86 self.update_list(dataOut)
87 87 data = {
88 88 'pow': 10*numpy.log10(dataOut.data_pow)
89 89 }
90 90 try:
91 91 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
92 92 except:
93 pass
93 pass
94 94 return data, {}
95 95
96 96 class SpectralWidthPlot(RTIPlot):
97 97 '''
98 98 Plot for Spectral Width Data (2nd moment)
99 99 '''
100 100
101 101 CODE = 'width'
102 102 colormap = 'jet'
103 103
104 104 def update(self, dataOut):
105 105 self.update_list(dataOut)
106 106 data = {
107 107 'width': dataOut.data_width
108 108 }
109 109 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
110 110 return data, {}
111 111
112 112 class SkyMapPlot(Plot):
113 113 '''
114 114 Plot for meteors detection data
115 115 '''
116 116
117 117 CODE = 'param'
118 118
119 119 def setup(self):
120 120
121 121 self.ncols = 1
122 122 self.nrows = 1
123 123 self.width = 7.2
124 124 self.height = 7.2
125 125 self.nplots = 1
126 126 self.xlabel = 'Zonal Zenith Angle (deg)'
127 127 self.ylabel = 'Meridional Zenith Angle (deg)'
128 128 self.polar = True
129 129 self.ymin = -180
130 130 self.ymax = 180
131 131 self.colorbar = False
132 132
133 133 def plot(self):
134 134
135 135 arrayParameters = numpy.concatenate(self.data['param'])
136 136 error = arrayParameters[:, -1]
137 137 indValid = numpy.where(error == 0)[0]
138 138 finalMeteor = arrayParameters[indValid, :]
139 139 finalAzimuth = finalMeteor[:, 3]
140 140 finalZenith = finalMeteor[:, 4]
141 141
142 142 x = finalAzimuth * numpy.pi / 180
143 143 y = finalZenith
144 144
145 145 ax = self.axes[0]
146 146
147 147 if ax.firsttime:
148 148 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
149 149 else:
150 150 ax.plot.set_data(x, y)
151 151
152 152 dt1 = self.getDateTime(self.data.min_time).strftime('%y/%m/%d %H:%M:%S')
153 153 dt2 = self.getDateTime(self.data.max_time).strftime('%y/%m/%d %H:%M:%S')
154 154 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
155 155 dt2,
156 156 len(x))
157 157 self.titles[0] = title
158 158
159 159
160 160 class GenericRTIPlot(Plot):
161 161 '''
162 162 Plot for data_xxxx object
163 163 '''
164 164
165 165 CODE = 'param'
166 166 colormap = 'viridis'
167 167 plot_type = 'pcolorbuffer'
168 168
169 169 def setup(self):
170 170 self.xaxis = 'time'
171 171 self.ncols = 1
172 172 self.nrows = self.data.shape('param')[0]
173 173 self.nplots = self.nrows
174 174 self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95, 'top': 0.95})
175 175
176 176 if not self.xlabel:
177 177 self.xlabel = 'Time'
178 178
179 179 self.ylabel = 'Height [km]'
180 180 if not self.titles:
181 181 self.titles = ['Param {}'.format(x) for x in range(self.nrows)]
182 182
183 183 def update(self, dataOut):
184 184
185 185 data = {
186 186 'param' : numpy.concatenate([getattr(dataOut, attr) for attr in self.attr_data], axis=0)
187 187 }
188 188
189 189 meta = {}
190 190
191 191 return data, meta
192 192
193 193 def plot(self):
194 194 # self.data.normalize_heights()
195 195 self.x = self.data.times
196 196 self.y = self.data.yrange
197 197 self.z = self.data['param']
198 198
199 199 self.z = numpy.ma.masked_invalid(self.z)
200 200
201 201 if self.decimation is None:
202 202 x, y, z = self.fill_gaps(self.x, self.y, self.z)
203 203 else:
204 204 x, y, z = self.fill_gaps(*self.decimate())
205 205
206 206 for n, ax in enumerate(self.axes):
207 207
208 208 self.zmax = self.zmax if self.zmax is not None else numpy.max(
209 209 self.z[n])
210 210 self.zmin = self.zmin if self.zmin is not None else numpy.min(
211 211 self.z[n])
212 212
213 213 if ax.firsttime:
214 214 if self.zlimits is not None:
215 215 self.zmin, self.zmax = self.zlimits[n]
216 216
217 217 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
218 218 vmin=self.zmin,
219 219 vmax=self.zmax,
220 220 cmap=self.cmaps[n]
221 221 )
222 222 else:
223 223 if self.zlimits is not None:
224 224 self.zmin, self.zmax = self.zlimits[n]
225 225 ax.collections.remove(ax.collections[0])
226 226 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
227 227 vmin=self.zmin,
228 228 vmax=self.zmax,
229 229 cmap=self.cmaps[n]
230 230 )
231 231
232 232
233 233 class PolarMapPlot(Plot):
234 234 '''
235 235 Plot for weather radar
236 236 '''
237 237
238 238 CODE = 'param'
239 239 colormap = 'seismic'
240 240
241 241 def setup(self):
242 242 self.ncols = 1
243 243 self.nrows = 1
244 244 self.width = 9
245 245 self.height = 8
246 246 self.mode = self.data.meta['mode']
247 247 if self.channels is not None:
248 248 self.nplots = len(self.channels)
249 249 self.nrows = len(self.channels)
250 250 else:
251 251 self.nplots = self.data.shape(self.CODE)[0]
252 252 self.nrows = self.nplots
253 253 self.channels = list(range(self.nplots))
254 254 if self.mode == 'E':
255 255 self.xlabel = 'Longitude'
256 256 self.ylabel = 'Latitude'
257 257 else:
258 258 self.xlabel = 'Range (km)'
259 259 self.ylabel = 'Height (km)'
260 260 self.bgcolor = 'white'
261 261 self.cb_labels = self.data.meta['units']
262 262 self.lat = self.data.meta['latitude']
263 263 self.lon = self.data.meta['longitude']
264 264 self.xmin, self.xmax = float(
265 265 km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
266 266 self.ymin, self.ymax = float(
267 267 km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
268 268 # self.polar = True
269 269
270 270 def plot(self):
271 271
272 272 for n, ax in enumerate(self.axes):
273 273 data = self.data['param'][self.channels[n]]
274 274
275 275 zeniths = numpy.linspace(
276 276 0, self.data.meta['max_range'], data.shape[1])
277 277 if self.mode == 'E':
278 278 azimuths = -numpy.radians(self.data.yrange)+numpy.pi/2
279 279 r, theta = numpy.meshgrid(zeniths, azimuths)
280 280 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(
281 281 theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
282 282 x = km2deg(x) + self.lon
283 283 y = km2deg(y) + self.lat
284 284 else:
285 285 azimuths = numpy.radians(self.data.yrange)
286 286 r, theta = numpy.meshgrid(zeniths, azimuths)
287 287 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
288 288 self.y = zeniths
289 289
290 290 if ax.firsttime:
291 291 if self.zlimits is not None:
292 292 self.zmin, self.zmax = self.zlimits[n]
293 293 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
294 294 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
295 295 vmin=self.zmin,
296 296 vmax=self.zmax,
297 297 cmap=self.cmaps[n])
298 298 else:
299 299 if self.zlimits is not None:
300 300 self.zmin, self.zmax = self.zlimits[n]
301 301 ax.collections.remove(ax.collections[0])
302 302 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
303 303 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
304 304 vmin=self.zmin,
305 305 vmax=self.zmax,
306 306 cmap=self.cmaps[n])
307 307
308 308 if self.mode == 'A':
309 309 continue
310 310
311 311 # plot district names
312 312 f = open('/data/workspace/schain_scripts/distrito.csv')
313 313 for line in f:
314 314 label, lon, lat = [s.strip() for s in line.split(',') if s]
315 315 lat = float(lat)
316 316 lon = float(lon)
317 317 # ax.plot(lon, lat, '.b', ms=2)
318 318 ax.text(lon, lat, label.decode('utf8'), ha='center',
319 319 va='bottom', size='8', color='black')
320 320
321 321 # plot limites
322 322 limites = []
323 323 tmp = []
324 324 for line in open('/data/workspace/schain_scripts/lima.csv'):
325 325 if '#' in line:
326 326 if tmp:
327 327 limites.append(tmp)
328 328 tmp = []
329 329 continue
330 330 values = line.strip().split(',')
331 331 tmp.append((float(values[0]), float(values[1])))
332 332 for points in limites:
333 333 ax.add_patch(
334 334 Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
335 335
336 336 # plot Cuencas
337 337 for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
338 338 f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
339 339 values = [line.strip().split(',') for line in f]
340 340 points = [(float(s[0]), float(s[1])) for s in values]
341 341 ax.add_patch(Polygon(points, ec='b', fc='none'))
342 342
343 343 # plot grid
344 344 for r in (15, 30, 45, 60):
345 345 ax.add_artist(plt.Circle((self.lon, self.lat),
346 346 km2deg(r), color='0.6', fill=False, lw=0.2))
347 347 ax.text(
348 348 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
349 349 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
350 350 '{}km'.format(r),
351 351 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
352 352
353 353 if self.mode == 'E':
354 354 title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
355 355 label = 'E{:02d}'.format(int(self.data.meta['elevation']))
356 356 else:
357 357 title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
358 358 label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
359 359
360 360 self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
361 361 self.titles = ['{} {}'.format(
362 362 self.data.parameters[x], title) for x in self.channels]
Show file before | Show file after | Hide comments
@@ -1,1180 +1,1187
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Classes to plot Spectra data
6 6
7 7 """
8 8
9 9 import os
10 10 import numpy
11 11
12 12 from schainpy.model.graphics.jroplot_base import Plot, plt, log
13 13 from itertools import combinations
14 14 from matplotlib.ticker import LinearLocator
15 15
16 16 class SpectraPlot(Plot):
17 17 '''
18 18 Plot for Spectra data
19 19 '''
20 20
21 21 CODE = 'spc'
22 22 colormap = 'jet'
23 23 plot_type = 'pcolor'
24 24 buffering = False
25 25 channelList = []
26 26 elevationList = []
27 27 azimuthList = []
28 28
29 29 def setup(self):
30 30
31 31 self.nplots = len(self.data.channels)
32 32 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
33 33 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
34 34 self.height = 3.4 * self.nrows
35 35
36 36 self.cb_label = 'dB'
37 37 if self.showprofile:
38 38 self.width = 5.2 * self.ncols
39 39 else:
40 40 self.width = 4.2* self.ncols
41 41 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.12})
42 42 self.ylabel = 'Range [km]'
43 43
44 44
45 45 def update_list(self,dataOut):
46 46 if len(self.channelList) == 0:
47 47 self.channelList = dataOut.channelList
48 48 if len(self.elevationList) == 0:
49 49 self.elevationList = dataOut.elevationList
50 50 if len(self.azimuthList) == 0:
51 51 self.azimuthList = dataOut.azimuthList
52 52
53 53 def update(self, dataOut):
54 54
55 55 self.update_list(dataOut)
56 56 data = {}
57 57 meta = {}
58 58 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
59 59 data['spc'] = spc
60 60 data['rti'] = dataOut.getPower()
61 #data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
61 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
62 noise = 10*numpy.log10(dataOut.getNoise()/float(norm))
63 data['noise'] = noise[0]
64
62 65 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
63 66 if self.CODE == 'spc_moments':
64 67 data['moments'] = dataOut.moments
65 68
66 69 return data, meta
67 70
68 71 def plot(self):
69 72 if self.xaxis == "frequency":
70 73 x = self.data.xrange[0]
71 74 self.xlabel = "Frequency (kHz)"
72 75 elif self.xaxis == "time":
73 76 x = self.data.xrange[1]
74 77 self.xlabel = "Time (ms)"
75 78 else:
76 79 x = self.data.xrange[2]
77 80 self.xlabel = "Velocity (m/s)"
78 81
79 82 if self.CODE == 'spc_moments':
80 83 x = self.data.xrange[2]
81 84 self.xlabel = "Velocity (m/s)"
82 85
83 86 self.titles = []
84 87 y = self.data.yrange
85 88 self.y = y
86 89
87 90 data = self.data[-1]
88 91 z = data['spc']
89 92 #print(z.shape, x.shape, y.shape)
90 93 for n, ax in enumerate(self.axes):
91 #noise = data['noise'][n]
92 noise=62
94 noise = self.data['noise'][n]
95 #print(noise)
93 96 if self.CODE == 'spc_moments':
94 97 mean = data['moments'][n, 1]
95 98 if ax.firsttime:
96 99 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
97 100 self.xmin = self.xmin if self.xmin else -self.xmax
98 101 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
99 102 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
100 103 ax.plt = ax.pcolormesh(x, y, z[n].T,
101 104 vmin=self.zmin,
102 105 vmax=self.zmax,
103 106 cmap=plt.get_cmap(self.colormap)
104 107 )
105 108
106 109 if self.showprofile:
107 110 ax.plt_profile = self.pf_axes[n].plot(
108 111 data['rti'][n], y)[0]
109 112 # ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
110 113 # color="k", linestyle="dashed", lw=1)[0]
111 114 if self.CODE == 'spc_moments':
112 115 ax.plt_mean = ax.plot(mean, y, color='k')[0]
113 116 else:
114 117 ax.plt.set_array(z[n].T.ravel())
115 118 if self.showprofile:
116 119 ax.plt_profile.set_data(data['rti'][n], y)
117 120 #ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
118 121 if self.CODE == 'spc_moments':
119 122 ax.plt_mean.set_data(mean, y)
120 123 if len(self.azimuthList) > 0 and len(self.elevationList) > 0:
121 124 self.titles.append('CH {}: {:2.1f}elv {:2.1f}az {:3.2f}dB'.format(self.channelList[n], noise, self.elevationList[n], self.azimuthList[n]))
122 125 else:
123 126 self.titles.append('CH {}: {:3.2f}dB'.format(self.channelList[n], noise))
124 127
125 128
126 129 class CrossSpectraPlot(Plot):
127 130
128 131 CODE = 'cspc'
129 132 colormap = 'jet'
130 133 plot_type = 'pcolor'
131 134 zmin_coh = None
132 135 zmax_coh = None
133 136 zmin_phase = None
134 137 zmax_phase = None
135 138 realChannels = None
136 139 crossPairs = None
137 140
138 141 def setup(self):
139 142
140 143 self.ncols = 4
141 144 self.nplots = len(self.data.pairs) * 2
142 145 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
143 146 self.width = 3.1 * self.ncols
144 147 self.height = 2.6 * self.nrows
145 148 self.ylabel = 'Range [km]'
146 149 self.showprofile = False
147 150 self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
148 151
149 152 def update(self, dataOut):
150 153
151 154 data = {}
152 155 meta = {}
153 156
154 157 spc = dataOut.data_spc
155 158 cspc = dataOut.data_cspc
156 159 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
157 160 rawPairs = list(combinations(list(range(dataOut.nChannels)), 2))
158 161 meta['pairs'] = rawPairs
159 162
160 163 if self.crossPairs == None:
161 164 self.crossPairs = dataOut.pairsList
162 165
163 166 tmp = []
164 167
165 168 for n, pair in enumerate(meta['pairs']):
166 169
167 170 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
168 171 coh = numpy.abs(out)
169 172 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
170 173 tmp.append(coh)
171 174 tmp.append(phase)
172 175
173 176 data['cspc'] = numpy.array(tmp)
174 177
175 178 return data, meta
176 179
177 180 def plot(self):
178 181
179 182 if self.xaxis == "frequency":
180 183 x = self.data.xrange[0]
181 184 self.xlabel = "Frequency (kHz)"
182 185 elif self.xaxis == "time":
183 186 x = self.data.xrange[1]
184 187 self.xlabel = "Time (ms)"
185 188 else:
186 189 x = self.data.xrange[2]
187 190 self.xlabel = "Velocity (m/s)"
188 191
189 192 self.titles = []
190 193
191 194 y = self.data.yrange
192 195 self.y = y
193 196
194 197 data = self.data[-1]
195 198 cspc = data['cspc']
196 199
197 200 for n in range(len(self.data.pairs)):
198 201
199 202 pair = self.crossPairs[n]
200 203
201 204 coh = cspc[n*2]
202 205 phase = cspc[n*2+1]
203 206 ax = self.axes[2 * n]
204 207
205 208 if ax.firsttime:
206 209 ax.plt = ax.pcolormesh(x, y, coh.T,
207 210 vmin=self.zmin_coh,
208 211 vmax=self.zmax_coh,
209 212 cmap=plt.get_cmap(self.colormap_coh)
210 213 )
211 214 else:
212 215 ax.plt.set_array(coh.T.ravel())
213 216 self.titles.append(
214 217 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
215 218
216 219 ax = self.axes[2 * n + 1]
217 220 if ax.firsttime:
218 221 ax.plt = ax.pcolormesh(x, y, phase.T,
219 222 vmin=-180,
220 223 vmax=180,
221 224 cmap=plt.get_cmap(self.colormap_phase)
222 225 )
223 226 else:
224 227 ax.plt.set_array(phase.T.ravel())
225 228
226 229 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
227 230
228 231
229 232 class RTIPlot(Plot):
230 233 '''
231 234 Plot for RTI data
232 235 '''
233 236
234 237 CODE = 'rti'
235 238 colormap = 'jet'
236 239 plot_type = 'pcolorbuffer'
237 240 titles = None
238 241 channelList = []
239 242 elevationList = []
240 243 azimuthList = []
241 244
242 245 def setup(self):
243 246 self.xaxis = 'time'
244 247 self.ncols = 1
245 248 #print("dataChannels ",self.data.channels)
246 249 self.nrows = len(self.data.channels)
247 250 self.nplots = len(self.data.channels)
248 251 self.ylabel = 'Range [km]'
249 252 self.xlabel = 'Time'
250 253 self.cb_label = 'dB'
251 254 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
252 255 self.titles = ['{} Channel {}'.format(
253 256 self.CODE.upper(), x) for x in range(self.nplots)]
254 257
255 258 def update_list(self,dataOut):
256 259
257 260 if len(self.channelList) == 0:
258 261 self.channelList = dataOut.channelList
259 262 if len(self.elevationList) == 0:
260 263 self.elevationList = dataOut.elevationList
261 264 if len(self.azimuthList) == 0:
262 265 self.azimuthList = dataOut.azimuthList
263 266
264 267
265 268 def update(self, dataOut):
266 269 if len(self.channelList) == 0:
267 270 self.update_list(dataOut)
268 271 data = {}
269 272 meta = {}
270 273 data['rti'] = dataOut.getPower()
271 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
274
275 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
276 noise = 10*numpy.log10(dataOut.getNoise()/float(norm))
277 data['noise'] = noise
278
272 279 return data, meta
273 280
274 281 def plot(self):
275 282
276 283 self.x = self.data.times
277 284 self.y = self.data.yrange
278 285 #print(" x, y: ",self.x, self.y)
279 286 self.z = self.data[self.CODE]
280 287 self.z = numpy.array(self.z, dtype=float)
281 288 self.z = numpy.ma.masked_invalid(self.z)
282 289
283 290 try:
284 291 if self.channelList != None:
285 292 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
286 293 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
287 294 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
288 295 else:
289 296 self.titles = ['{} Channel {}'.format(
290 297 self.CODE.upper(), x) for x in self.channelList]
291 298 except:
292 299 if self.channelList.any() != None:
293 300
294 301 self.titles = ['{} Channel {}'.format(
295 302 self.CODE.upper(), x) for x in self.channelList]
296 303
297 304 if self.decimation is None:
298 305 x, y, z = self.fill_gaps(self.x, self.y, self.z)
299 306 else:
300 307 x, y, z = self.fill_gaps(*self.decimate())
301 308
302 309 #dummy_var = self.axes #Extrañamente esto actualiza el valor axes
303 310 for n, ax in enumerate(self.axes):
304 311 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
305 312 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
306 313 data = self.data[-1]
307 314
308 315 if ax.firsttime:
309 316 ax.plt = ax.pcolormesh(x, y, z[n].T,
310 317 vmin=self.zmin,
311 318 vmax=self.zmax,
312 319 cmap=plt.get_cmap(self.colormap)
313 320 )
314 321 if self.showprofile:
315 322 ax.plot_profile = self.pf_axes[n].plot(data[self.CODE][n], self.y)[0]
316 323 if "noise" in self.data:
317 324
318 325 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(data['noise'][n], len(self.y)), self.y,
319 326 color="k", linestyle="dashed", lw=1)[0]
320 327 else:
321 328 ax.collections.remove(ax.collections[0])
322 329 ax.plt = ax.pcolormesh(x, y, z[n].T,
323 330 vmin=self.zmin,
324 331 vmax=self.zmax,
325 332 cmap=plt.get_cmap(self.colormap)
326 333 )
327 334 if self.showprofile:
328 335 ax.plot_profile.set_data(data[self.CODE][n], self.y)
329 336 if "noise" in self.data:
330
331 ax.plot_noise.set_data(numpy.repeat(
332 data['noise'][n], len(self.y)), self.y)
337 ax.plot_noise.set_data(numpy.repeat(data['noise'][n], len(self.y)), self.y)
333 338
334 339
335 340 class CoherencePlot(RTIPlot):
336 341 '''
337 342 Plot for Coherence data
338 343 '''
339 344
340 345 CODE = 'coh'
341 346
342 347 def setup(self):
343 348 self.xaxis = 'time'
344 349 self.ncols = 1
345 350 self.nrows = len(self.data.pairs)
346 351 self.nplots = len(self.data.pairs)
347 352 self.ylabel = 'Range [km]'
348 353 self.xlabel = 'Time'
349 354 self.plots_adjust.update({'hspace':0.6, 'left': 0.1, 'bottom': 0.1,'right':0.95})
350 355 if self.CODE == 'coh':
351 356 self.cb_label = ''
352 357 self.titles = [
353 358 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
354 359 else:
355 360 self.cb_label = 'Degrees'
356 361 self.titles = [
357 362 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
358 363
359 364 def update(self, dataOut):
360 365 self.update_list(dataOut)
361 366 data = {}
362 367 meta = {}
363 368 data['coh'] = dataOut.getCoherence()
364 369 meta['pairs'] = dataOut.pairsList
365 370
366 371
367 372 return data, meta
368 373
369 374 class PhasePlot(CoherencePlot):
370 375 '''
371 376 Plot for Phase map data
372 377 '''
373 378
374 379 CODE = 'phase'
375 380 colormap = 'seismic'
376 381
377 382 def update(self, dataOut):
378 383
379 384 data = {}
380 385 meta = {}
381 386 data['phase'] = dataOut.getCoherence(phase=True)
382 387 meta['pairs'] = dataOut.pairsList
383 388
384 389 return data, meta
385 390
386 391 class NoisePlot(Plot):
387 392 '''
388 393 Plot for noise
389 394 '''
390 395
391 396 CODE = 'noise'
392 397 plot_type = 'scatterbuffer'
393 398
394 399 def setup(self):
395 400 self.xaxis = 'time'
396 401 self.ncols = 1
397 402 self.nrows = 1
398 403 self.nplots = 1
399 404 self.ylabel = 'Intensity [dB]'
400 405 self.xlabel = 'Time'
401 406 self.titles = ['Noise']
402 407 self.colorbar = False
403 408 self.plots_adjust.update({'right': 0.85 })
409 #if not self.titles:
410 self.titles = ['Noise Plot']
404 411
405 412 def update(self, dataOut):
406 413
407 414 data = {}
408 415 meta = {}
409 noise = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor).reshape(dataOut.nChannels, 1)
416 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
417 #noise = 10*numpy.log10(dataOut.getNoise()/norm)
418 noise = 10*numpy.log10(dataOut.getNoise())
419 noise = noise.reshape(dataOut.nChannels, 1)
410 420 data['noise'] = noise
411 421 meta['yrange'] = numpy.array([])
412 422
413 423 return data, meta
414 424
415 425 def plot(self):
416 426
417 427 x = self.data.times
418 428 xmin = self.data.min_time
419 429 xmax = xmin + self.xrange * 60 * 60
420 430 Y = self.data['noise']
421 431
422 432 if self.axes[0].firsttime:
423 433 if self.ymin is None: self.ymin = numpy.nanmin(Y) - 5
424 434 if self.ymax is None: self.ymax = numpy.nanmax(Y) + 5
425 435 for ch in self.data.channels:
426 436 y = Y[ch]
427 437 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
428 438 plt.legend(bbox_to_anchor=(1.18, 1.0))
429 439 else:
430 440 for ch in self.data.channels:
431 441 y = Y[ch]
432 442 self.axes[0].lines[ch].set_data(x, y)
433 443
434 444
435 445 class PowerProfilePlot(Plot):
436 446
437 447 CODE = 'pow_profile'
438 448 plot_type = 'scatter'
439 449
440 450 def setup(self):
441 451
442 452 self.ncols = 1
443 453 self.nrows = 1
444 454 self.nplots = 1
445 455 self.height = 4
446 456 self.width = 3
447 457 self.ylabel = 'Range [km]'
448 458 self.xlabel = 'Intensity [dB]'
449 459 self.titles = ['Power Profile']
450 460 self.colorbar = False
451 461
452 462 def update(self, dataOut):
453 463
454 464 data = {}
455 465 meta = {}
456 466 data[self.CODE] = dataOut.getPower()
457 467
458 468 return data, meta
459 469
460 470 def plot(self):
461 471
462 472 y = self.data.yrange
463 473 self.y = y
464 474
465 475 x = self.data[-1][self.CODE]
466 476
467 477 if self.xmin is None: self.xmin = numpy.nanmin(x)*0.9
468 478 if self.xmax is None: self.xmax = numpy.nanmax(x)*1.1
469 479
470 480 if self.axes[0].firsttime:
471 481 for ch in self.data.channels:
472 482 self.axes[0].plot(x[ch], y, lw=1, label='Ch{}'.format(ch))
473 483 plt.legend()
474 484 else:
475 485 for ch in self.data.channels:
476 486 self.axes[0].lines[ch].set_data(x[ch], y)
477 487
478 488
479 489 class SpectraCutPlot(Plot):
480 490
481 491 CODE = 'spc_cut'
482 492 plot_type = 'scatter'
483 493 buffering = False
484 494 heights = []
485 495 channelList = []
486 496 maintitle = "Spectra Cuts"
487 497 flag_setIndex = False
488 498
489 499 def setup(self):
490 500
491 501 self.nplots = len(self.data.channels)
492 502 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
493 503 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
494 self.width = 4.2 * self.ncols + 2.5
504 self.width = 4.5 * self.ncols + 2.5
495 505 self.height = 4.8 * self.nrows
496 506 self.ylabel = 'Power [dB]'
497 507 self.colorbar = False
498 self.plots_adjust.update({'left':0.05, 'hspace':0.3, 'right': 0.9, 'bottom':0.08})
508 self.plots_adjust.update({'left':0.1, 'hspace':0.3, 'right': 0.9, 'bottom':0.08})
499 509
500 510 if len(self.selectedHeightsList) > 0:
501 511 self.maintitle = "Spectra Cut"# for %d km " %(int(self.selectedHeight))
502 512
503 513
504 514
505 515 def update(self, dataOut):
506 516 if len(self.channelList) == 0:
507 517 self.channelList = dataOut.channelList
508 518
509 519 self.heights = dataOut.heightList
510 520 #print("sels: ",self.selectedHeightsList)
511 521 if len(self.selectedHeightsList)>0 and not self.flag_setIndex:
512 522
513 523 for sel_height in self.selectedHeightsList:
514 524 index_list = numpy.where(self.heights >= sel_height)
515 525 index_list = index_list[0]
516 526 self.height_index.append(index_list[0])
517 527 #print("sels i:"", self.height_index)
518 528 self.flag_setIndex = True
519 529 #print(self.height_index)
520 530 data = {}
521 531 meta = {}
522 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
532
533 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
534 n0 = 10*numpy.log10(dataOut.getNoise()/float(norm))
535
536 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor) - n0
523 537
524 538 data['spc'] = spc
525 539 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
526 540
527 541 return data, meta
528 542
529 543 def plot(self):
530 544 if self.xaxis == "frequency":
531 545 x = self.data.xrange[0][1:]
532 546 self.xlabel = "Frequency (kHz)"
533 547 elif self.xaxis == "time":
534 548 x = self.data.xrange[1]
535 549 self.xlabel = "Time (ms)"
536 550 else:
537 551 x = self.data.xrange[2]
538 552 self.xlabel = "Velocity (m/s)"
539 553
540 554 self.titles = []
541 555
542 556 y = self.data.yrange
543 557 z = self.data[-1]['spc']
544 558 #print(z.shape)
545 559 if len(self.height_index) > 0:
546 560 index = self.height_index
547 561 else:
548 562 index = numpy.arange(0, len(y), int((len(y))/9))
549 563 #print("inde x ", index, self.axes)
550 564
551 565 for n, ax in enumerate(self.axes):
552 566
553 567 if ax.firsttime:
554 568
555 569
556 570 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
557 571 self.xmin = self.xmin if self.xmin else -self.xmax
558 572 self.ymin = self.ymin if self.ymin else numpy.nanmin(z)
559 573 self.ymax = self.ymax if self.ymax else numpy.nanmax(z)
560 574
561 575
562 576 ax.plt = ax.plot(x, z[n, :, index].T)
563 577 labels = ['Range = {:2.1f}km'.format(y[i]) for i in index]
564 578 self.figures[0].legend(ax.plt, labels, loc='center right', prop={'size': 8})
565 579 ax.minorticks_on()
566 580 ax.grid(which='major', axis='both')
567 581 ax.grid(which='minor', axis='x')
568 582 else:
569 583 for i, line in enumerate(ax.plt):
570 584 line.set_data(x, z[n, :, index[i]])
571 585
572 586
573 587 self.titles.append('CH {}'.format(self.channelList[n]))
574 588 plt.suptitle(self.maintitle, fontsize=10)
575 589
576 590
577 591 class BeaconPhase(Plot):
578 592
579 593 __isConfig = None
580 594 __nsubplots = None
581 595
582 596 PREFIX = 'beacon_phase'
583 597
584 598 def __init__(self):
585 599 Plot.__init__(self)
586 600 self.timerange = 24*60*60
587 601 self.isConfig = False
588 602 self.__nsubplots = 1
589 603 self.counter_imagwr = 0
590 604 self.WIDTH = 800
591 605 self.HEIGHT = 400
592 606 self.WIDTHPROF = 120
593 607 self.HEIGHTPROF = 0
594 608 self.xdata = None
595 609 self.ydata = None
596 610
597 611 self.PLOT_CODE = BEACON_CODE
598 612
599 613 self.FTP_WEI = None
600 614 self.EXP_CODE = None
601 615 self.SUB_EXP_CODE = None
602 616 self.PLOT_POS = None
603 617
604 618 self.filename_phase = None
605 619
606 620 self.figfile = None
607 621
608 622 self.xmin = None
609 623 self.xmax = None
610 624
611 625 def getSubplots(self):
612 626
613 627 ncol = 1
614 628 nrow = 1
615 629
616 630 return nrow, ncol
617 631
618 632 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
619 633
620 634 self.__showprofile = showprofile
621 635 self.nplots = nplots
622 636
623 637 ncolspan = 7
624 638 colspan = 6
625 639 self.__nsubplots = 2
626 640
627 641 self.createFigure(id = id,
628 642 wintitle = wintitle,
629 643 widthplot = self.WIDTH+self.WIDTHPROF,
630 644 heightplot = self.HEIGHT+self.HEIGHTPROF,
631 645 show=show)
632 646
633 647 nrow, ncol = self.getSubplots()
634 648
635 649 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
636 650
637 651 def save_phase(self, filename_phase):
638 652 f = open(filename_phase,'w+')
639 653 f.write('\n\n')
640 654 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
641 655 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
642 656 f.close()
643 657
644 658 def save_data(self, filename_phase, data, data_datetime):
645 659 f=open(filename_phase,'a')
646 660 timetuple_data = data_datetime.timetuple()
647 661 day = str(timetuple_data.tm_mday)
648 662 month = str(timetuple_data.tm_mon)
649 663 year = str(timetuple_data.tm_year)
650 664 hour = str(timetuple_data.tm_hour)
651 665 minute = str(timetuple_data.tm_min)
652 666 second = str(timetuple_data.tm_sec)
653 667 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
654 668 f.close()
655 669
656 670 def plot(self):
657 671 log.warning('TODO: Not yet implemented...')
658 672
659 673 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
660 674 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
661 675 timerange=None,
662 676 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
663 677 server=None, folder=None, username=None, password=None,
664 678 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
665 679
666 680 if dataOut.flagNoData:
667 681 return dataOut
668 682
669 683 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
670 684 return
671 685
672 686 if pairsList == None:
673 687 pairsIndexList = dataOut.pairsIndexList[:10]
674 688 else:
675 689 pairsIndexList = []
676 690 for pair in pairsList:
677 691 if pair not in dataOut.pairsList:
678 692 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
679 693 pairsIndexList.append(dataOut.pairsList.index(pair))
680 694
681 695 if pairsIndexList == []:
682 696 return
683 697
684 698 # if len(pairsIndexList) > 4:
685 699 # pairsIndexList = pairsIndexList[0:4]
686 700
687 701 hmin_index = None
688 702 hmax_index = None
689 703
690 704 if hmin != None and hmax != None:
691 705 indexes = numpy.arange(dataOut.nHeights)
692 706 hmin_list = indexes[dataOut.heightList >= hmin]
693 707 hmax_list = indexes[dataOut.heightList <= hmax]
694 708
695 709 if hmin_list.any():
696 710 hmin_index = hmin_list[0]
697 711
698 712 if hmax_list.any():
699 713 hmax_index = hmax_list[-1]+1
700 714
701 715 x = dataOut.getTimeRange()
702 716
703 717 thisDatetime = dataOut.datatime
704 718
705 719 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
706 720 xlabel = "Local Time"
707 721 ylabel = "Phase (degrees)"
708 722
709 723 update_figfile = False
710 724
711 725 nplots = len(pairsIndexList)
712 726 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
713 727 phase_beacon = numpy.zeros(len(pairsIndexList))
714 728 for i in range(nplots):
715 729 pair = dataOut.pairsList[pairsIndexList[i]]
716 730 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
717 731 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
718 732 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
719 733 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
720 734 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
721 735
722 736 if dataOut.beacon_heiIndexList:
723 737 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
724 738 else:
725 739 phase_beacon[i] = numpy.average(phase)
726 740
727 741 if not self.isConfig:
728 742
729 743 nplots = len(pairsIndexList)
730 744
731 745 self.setup(id=id,
732 746 nplots=nplots,
733 747 wintitle=wintitle,
734 748 showprofile=showprofile,
735 749 show=show)
736 750
737 751 if timerange != None:
738 752 self.timerange = timerange
739 753
740 754 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
741 755
742 756 if ymin == None: ymin = 0
743 757 if ymax == None: ymax = 360
744 758
745 759 self.FTP_WEI = ftp_wei
746 760 self.EXP_CODE = exp_code
747 761 self.SUB_EXP_CODE = sub_exp_code
748 762 self.PLOT_POS = plot_pos
749 763
750 764 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
751 765 self.isConfig = True
752 766 self.figfile = figfile
753 767 self.xdata = numpy.array([])
754 768 self.ydata = numpy.array([])
755 769
756 770 update_figfile = True
757 771
758 772 #open file beacon phase
759 773 path = '%s%03d' %(self.PREFIX, self.id)
760 774 beacon_file = os.path.join(path,'%s.txt'%self.name)
761 775 self.filename_phase = os.path.join(figpath,beacon_file)
762 776 #self.save_phase(self.filename_phase)
763 777
764 778
765 779 #store data beacon phase
766 780 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
767 781
768 782 self.setWinTitle(title)
769 783
770 784
771 785 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
772 786
773 787 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
774 788
775 789 axes = self.axesList[0]
776 790
777 791 self.xdata = numpy.hstack((self.xdata, x[0:1]))
778 792
779 793 if len(self.ydata)==0:
780 794 self.ydata = phase_beacon.reshape(-1,1)
781 795 else:
782 796 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
783 797
784 798
785 799 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
786 800 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
787 801 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
788 802 XAxisAsTime=True, grid='both'
789 803 )
790 804
791 805 self.draw()
792 806
793 807 if dataOut.ltctime >= self.xmax:
794 808 self.counter_imagwr = wr_period
795 809 self.isConfig = False
796 810 update_figfile = True
797 811
798 812 self.save(figpath=figpath,
799 813 figfile=figfile,
800 814 save=save,
801 815 ftp=ftp,
802 816 wr_period=wr_period,
803 817 thisDatetime=thisDatetime,
804 818 update_figfile=update_figfile)
805 819
806 820 return dataOut
807 821
808 822 class NoiselessSpectraPlot(Plot):
809 823 '''
810 824 Plot for Spectra data, subtracting
811 825 the noise in all channels, using for
812 826 amisr-14 data
813 827 '''
814 828
815 829 CODE = 'noiseless_spc'
816 830 colormap = 'nipy_spectral'
817 831 plot_type = 'pcolor'
818 832 buffering = False
819 833 channelList = []
820 834 last_noise = None
821 835
822 836 def setup(self):
823 837
824 838 self.nplots = len(self.data.channels)
825 839 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
826 840 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
827 self.height = 2.6 * self.nrows
841 self.height = 3.5 * self.nrows
828 842
829 843 self.cb_label = 'dB'
830 844 if self.showprofile:
831 self.width = 4 * self.ncols
845 self.width = 5.8 * self.ncols
832 846 else:
833 self.width = 3.5 * self.ncols
834 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.08})
847 self.width = 4.8* self.ncols
848 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.92, 'bottom': 0.12})
849
835 850 self.ylabel = 'Range [km]'
836 851
837 852
838 853 def update_list(self,dataOut):
839 854 if len(self.channelList) == 0:
840 855 self.channelList = dataOut.channelList
841 856
842 857 def update(self, dataOut):
843 858
844 859 self.update_list(dataOut)
845 860 data = {}
846 861 meta = {}
847 862
848 863 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
849 864 n0 = 10*numpy.log10(dataOut.getNoise()/float(norm))
850 865
851 if self.last_noise == None:
852 self.last_noise = n0
853 else:
854 n0 = (n0*0.2 + self.last_noise*0.8)
855 self.last_noise = n0
856 866
857 867 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
858 868
859 869 data['spc'] = spc - n0
860 870 data['rti'] = dataOut.getPower() - n0
861 871
862 #data['noise'] = noise
872 # data['noise'] = noise
863 873 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
864 874
865 875 return data, meta
866 876
867 877 def plot(self):
868 878 if self.xaxis == "frequency":
869 879 x = self.data.xrange[0]
870 880 self.xlabel = "Frequency (kHz)"
871 881 elif self.xaxis == "time":
872 882 x = self.data.xrange[1]
873 883 self.xlabel = "Time (ms)"
874 884 else:
875 885 x = self.data.xrange[2]
876 886 self.xlabel = "Velocity (m/s)"
877 887
878 888 self.titles = []
879 889 y = self.data.yrange
880 890 self.y = y
881 891
882 892 data = self.data[-1]
883 893 z = data['spc']
884 894
885 895 for n, ax in enumerate(self.axes):
886 896 #noise = data['noise'][n]
887 897
888 898 if ax.firsttime:
889 899 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
890 900 self.xmin = self.xmin if self.xmin else -self.xmax
891 901 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
892 902 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
893 903 ax.plt = ax.pcolormesh(x, y, z[n].T,
894 904 vmin=self.zmin,
895 905 vmax=self.zmax,
896 906 cmap=plt.get_cmap(self.colormap)
897 907 )
898 908
899 909 if self.showprofile:
900 910 ax.plt_profile = self.pf_axes[n].plot(
901 911 data['rti'][n], y)[0]
902 912
903 913
904 914 else:
905 915 ax.plt.set_array(z[n].T.ravel())
906 916 if self.showprofile:
907 917 ax.plt_profile.set_data(data['rti'][n], y)
908 918
909 919
910 920 self.titles.append('CH {}'.format(self.channelList[n]))
911 921
912 922
913 923 class NoiselessRTIPlot(Plot):
914 924 '''
915 925 Plot for RTI data
916 926 '''
917 927
918 928 CODE = 'noiseless_rti'
919 929 colormap = 'jet'
920 930 plot_type = 'pcolorbuffer'
921 931 titles = None
922 932 channelList = []
923 933 elevationList = []
924 934 azimuthList = []
925 935 last_noise = None
926 936
927 937 def setup(self):
928 938 self.xaxis = 'time'
929 939 self.ncols = 1
930 940 #print("dataChannels ",self.data.channels)
931 941 self.nrows = len(self.data.channels)
932 942 self.nplots = len(self.data.channels)
933 943 self.ylabel = 'Range [km]'
934 944 self.xlabel = 'Time'
935 945 self.cb_label = 'dB'
936 946 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
937 947 self.titles = ['{} Channel {}'.format(
938 948 self.CODE.upper(), x) for x in range(self.nplots)]
939 949
940 950 def update_list(self,dataOut):
941 951 if len(self.channelList) == 0:
942 952 self.channelList = dataOut.channelList
943 953 if len(self.elevationList) == 0:
944 954 self.elevationList = dataOut.elevationList
945 955 if len(self.azimuthList) == 0:
946 956 self.azimuthList = dataOut.azimuthList
947 957
948 958 def update(self, dataOut):
949 959 if len(self.channelList) == 0:
950 960 self.update_list(dataOut)
951 961 data = {}
952 962 meta = {}
953 963
954 964
955 965 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
956 966 n0 = 10*numpy.log10(dataOut.getNoise()/float(norm))
957 #print("noise: ",n0, dataOut.normFactor, norm, dataOut.nIncohInt, dataOut.max_nIncohInt)
958 if self.last_noise == None:
959 self.last_noise = n0
960 else:
961 n0 = (n0*0.2 + self.last_noise*0.8)
962 self.last_noise = n0
963 967
968 data['noise'] = n0[0]
964 969
965 970 data['noiseless_rti'] = dataOut.getPower() - n0
966 971
967 #data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
968 #print(noise)
969 972 return data, meta
970 973
971 974 def plot(self):
972 975
973 976 self.x = self.data.times
974 977 self.y = self.data.yrange
975 978 self.z = self.data['noiseless_rti']
976 979 self.z = numpy.array(self.z, dtype=float)
977 980 self.z = numpy.ma.masked_invalid(self.z)
978 981
982
979 983 try:
980 984 if self.channelList != None:
981 985 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
982 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
986 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
983 987 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
984 988 else:
985 self.titles = ['{} Channel {}'.format(
989 self.titles = ['{} Channel {}'.format(
986 990 self.CODE.upper(), x) for x in self.channelList]
987 991 except:
988 992 if self.channelList.any() != None:
989 993
990 994 self.titles = ['{} Channel {}'.format(
991 995 self.CODE.upper(), x) for x in self.channelList]
996
997
992 998 if self.decimation is None:
993 999 x, y, z = self.fill_gaps(self.x, self.y, self.z)
994 1000 else:
995 1001 x, y, z = self.fill_gaps(*self.decimate())
996 1002 dummy_var = self.axes #Extrañamente esto actualiza el valor axes
997 1003 #print("plot shapes ", z.shape, x.shape, y.shape)
998 1004 for n, ax in enumerate(self.axes):
1005
1006
999 1007 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
1000 1008 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
1001 1009 data = self.data[-1]
1002 1010 if ax.firsttime:
1003 1011 ax.plt = ax.pcolormesh(x, y, z[n].T,
1004 1012 vmin=self.zmin,
1005 1013 vmax=self.zmax,
1006 1014 cmap=plt.get_cmap(self.colormap)
1007 1015 )
1008 1016 if self.showprofile:
1009 1017 ax.plot_profile = self.pf_axes[n].plot(data['noiseless_rti'][n], self.y)[0]
1010 1018
1011 1019 else:
1012 1020 ax.collections.remove(ax.collections[0])
1013 1021 ax.plt = ax.pcolormesh(x, y, z[n].T,
1014 1022 vmin=self.zmin,
1015 1023 vmax=self.zmax,
1016 1024 cmap=plt.get_cmap(self.colormap)
1017 1025 )
1018 1026 if self.showprofile:
1019 1027 ax.plot_profile.set_data(data['noiseless_rti'][n], self.y)
1020 1028 # if "noise" in self.data:
1021 1029 # #ax.plot_noise.set_data(numpy.repeat(data['noise'][n], len(self.y)), self.y)
1022 1030 # ax.plot_noise.set_data(data['noise'][n], self.y)
1023 1031
1024 1032
1025 1033 class OutliersRTIPlot(Plot):
1026 1034 '''
1027 1035 Plot for data_xxxx object
1028 1036 '''
1029 1037
1030 CODE = 'outlier'
1038 CODE = 'outlier_rtc' # Range Time Counts
1031 1039 colormap = 'cool'
1032 1040 plot_type = 'pcolorbuffer'
1033 1041
1034 1042 def setup(self):
1035 1043 self.xaxis = 'time'
1036 1044 self.ncols = 1
1037 self.nrows = self.data.shape('outlier')[0]
1045 self.nrows = self.data.shape('outlier_rtc')[0]
1038 1046 self.nplots = self.nrows
1039 1047 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
1040 1048
1041 1049
1042 1050 if not self.xlabel:
1043 1051 self.xlabel = 'Time'
1044 1052
1045 1053 self.ylabel = 'Height [km]'
1046 1054 if not self.titles:
1047 1055 self.titles = ['Outliers Ch:{}'.format(x) for x in range(self.nrows)]
1048 1056
1049 1057 def update(self, dataOut):
1050 1058
1051 1059 data = {}
1052 data['outlier'] = dataOut.data_outlier
1060 data['outlier_rtc'] = dataOut.data_outlier
1053 1061
1054 1062 meta = {}
1055 1063
1056 1064 return data, meta
1057 1065
1058 1066 def plot(self):
1059 1067 # self.data.normalize_heights()
1060 1068 self.x = self.data.times
1061 1069 self.y = self.data.yrange
1062 self.z = self.data['outlier']
1070 self.z = self.data['outlier_rtc']
1063 1071
1064 1072 #self.z = numpy.ma.masked_invalid(self.z)
1065 1073
1066 1074 if self.decimation is None:
1067 1075 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1068 1076 else:
1069 1077 x, y, z = self.fill_gaps(*self.decimate())
1070 1078
1071 1079 for n, ax in enumerate(self.axes):
1072 1080
1073 1081 self.zmax = self.zmax if self.zmax is not None else numpy.max(
1074 1082 self.z[n])
1075 1083 self.zmin = self.zmin if self.zmin is not None else numpy.min(
1076 1084 self.z[n])
1077 1085 data = self.data[-1]
1078 1086 if ax.firsttime:
1079 1087 if self.zlimits is not None:
1080 1088 self.zmin, self.zmax = self.zlimits[n]
1081 1089
1082 1090 ax.plt = ax.pcolormesh(x, y, z[n].T,
1083 1091 vmin=self.zmin,
1084 1092 vmax=self.zmax,
1085 1093 cmap=self.cmaps[n]
1086 1094 )
1087 1095 if self.showprofile:
1088 ax.plot_profile = self.pf_axes[n].plot(data['outlier'][n], self.y)[0]
1096 ax.plot_profile = self.pf_axes[n].plot(data['outlier_rtc'][n], self.y)[0]
1089 1097 self.pf_axes[n].set_xlabel('')
1090 1098 else:
1091 1099 if self.zlimits is not None:
1092 1100 self.zmin, self.zmax = self.zlimits[n]
1093 1101 ax.collections.remove(ax.collections[0])
1094 1102 ax.plt = ax.pcolormesh(x, y, z[n].T ,
1095 1103 vmin=self.zmin,
1096 1104 vmax=self.zmax,
1097 1105 cmap=self.cmaps[n]
1098 1106 )
1099 1107 if self.showprofile:
1100 ax.plot_profile.set_data(data['outlier'][n], self.y)
1108 ax.plot_profile.set_data(data['outlier_rtc'][n], self.y)
1101 1109 self.pf_axes[n].set_xlabel('')
1102 1110
1103 # class NoiseRTIPlot(Plot):
1104 # '''
1105 # Plot for data_xxxx object
1106 # '''
1107 #
1108 # CODE = 'noise'
1109 # colormap = 'jet'
1110 # plot_type = 'pcolorbuffer'
1111 #
1112 # def setup(self):
1113 # self.xaxis = 'time'
1114 # self.ncols = 1
1115 # self.nrows = self.data.shape('noise')[0]
1116 # self.nplots = self.nrows
1117 # self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
1118 #
1119 #
1120 # if not self.xlabel:
1121 # self.xlabel = 'Time'
1122 #
1123 # self.ylabel = 'Height [km]'
1124 # if not self.titles:
1125 # self.titles = ['noise Ch:{}'.format(x) for x in range(self.nrows)]
1126 #
1127 # def update(self, dataOut):
1128 #
1129 # data = {}
1130 # norm = dataOut.max_nIncohInt*dataOut.nProfiles* dataOut.nCohInt*dataOut.windowOfFilter
1131 # print("max incoh: ",dataOut.max_nIncohInt )
1132 # n0 = 10*numpy.log10(dataOut.getNoise()/norm)
1133 # data['noise'] = n0
1134 #
1135 # meta = {}
1136 #
1137 # return data, meta
1138 #
1139 # def plot(self):
1140 # # self.data.normalize_heights()
1141 # self.x = self.data.times
1142 # self.y = self.data.yrange
1143 # self.z = self.data['noise']
1144 #
1145 # #self.z = numpy.ma.masked_invalid(self.z)
1146 #
1147 # if self.decimation is None:
1148 # x, y, z = self.fill_gaps(self.x, self.y, self.z)
1149 # else:
1150 # x, y, z = self.fill_gaps(*self.decimate())
1151 #
1152 # for n, ax in enumerate(self.axes):
1153 #
1154 # self.zmax = self.zmax if self.zmax is not None else numpy.max(
1155 # self.z[n])
1156 # self.zmin = self.zmin if self.zmin is not None else numpy.min(
1157 # self.z[n])
1158 # data = self.data[-1]
1159 # if ax.firsttime:
1160 # if self.zlimits is not None:
1161 # self.zmin, self.zmax = self.zlimits[n]
1162 #
1163 # ax.plt = ax.pcolormesh(x, y, z[n].T,
1164 # vmin=self.zmin,
1165 # vmax=self.zmax,
1166 # cmap=self.cmaps[n]
1167 # )
1168 # if self.showprofile:
1169 # ax.plot_profile = self.pf_axes[n].plot(data['noise'][n], self.y)[0]
1170 # else:
1171 # if self.zlimits is not None:
1172 # self.zmin, self.zmax = self.zlimits[n]
1173 # ax.collections.remove(ax.collections[0])
1174 # ax.plt = ax.pcolormesh(x, y, z[n].T ,
1175 # vmin=self.zmin,
1176 # vmax=self.zmax,
1177 # cmap=self.cmaps[n]
1178 # )
1179 # if self.showprofile:
1180 # ax.plot_profile.set_data(data['noise'][n], self.y)
1111 class NIncohIntRTIPlot(Plot):
1112 '''
1113 Plot for data_xxxx object
1114 '''
1115
1116 CODE = 'integrations_rtc' # Range Time Counts
1117 colormap = 'BuGn'
1118 plot_type = 'pcolorbuffer'
1119
1120 def setup(self):
1121 self.xaxis = 'time'
1122 self.ncols = 1
1123 self.nrows = self.data.shape('integrations_rtc')[0]
1124 self.nplots = self.nrows
1125 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
1126
1127
1128 if not self.xlabel:
1129 self.xlabel = 'Time'
1130
1131 self.ylabel = 'Height [km]'
1132 if not self.titles:
1133 self.titles = ['Integration Ch:{}'.format(x) for x in range(self.nrows)]
1134
1135 def update(self, dataOut):
1136
1137 data = {}
1138 data['integrations_rtc'] = dataOut.nIncohInt
1139
1140 meta = {}
1141
1142 return data, meta
1143
1144 def plot(self):
1145 # self.data.normalize_heights()
1146 self.x = self.data.times
1147 self.y = self.data.yrange
1148 self.z = self.data['integrations_rtc']
1149
1150 #self.z = numpy.ma.masked_invalid(self.z)
1151
1152 if self.decimation is None:
1153 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1154 else:
1155 x, y, z = self.fill_gaps(*self.decimate())
1156
1157 for n, ax in enumerate(self.axes):
1158
1159 self.zmax = self.zmax if self.zmax is not None else numpy.max(
1160 self.z[n])
1161 self.zmin = self.zmin if self.zmin is not None else numpy.min(
1162 self.z[n])
1163 data = self.data[-1]
1164 if ax.firsttime:
1165 if self.zlimits is not None:
1166 self.zmin, self.zmax = self.zlimits[n]
1167
1168 ax.plt = ax.pcolormesh(x, y, z[n].T,
1169 vmin=self.zmin,
1170 vmax=self.zmax,
1171 cmap=self.cmaps[n]
1172 )
1173 if self.showprofile:
1174 ax.plot_profile = self.pf_axes[n].plot(data['integrations_rtc'][n], self.y)[0]
1175 self.pf_axes[n].set_xlabel('')
1176 else:
1177 if self.zlimits is not None:
1178 self.zmin, self.zmax = self.zlimits[n]
1179 ax.collections.remove(ax.collections[0])
1180 ax.plt = ax.pcolormesh(x, y, z[n].T ,
1181 vmin=self.zmin,
1182 vmax=self.zmax,
1183 cmap=self.cmaps[n]
1184 )
1185 if self.showprofile:
1186 ax.plot_profile.set_data(data['integrations_rtc'][n], self.y)
1187 self.pf_axes[n].set_xlabel('')
Show file before | Show file after | Hide comments
@@ -1,685 +1,686
1 1 import os
2 2 import time
3 3 import datetime
4 4
5 5 import numpy
6 6 import h5py
7 7
8 8 import schainpy.admin
9 9 from schainpy.model.data.jrodata import *
10 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 11 from schainpy.model.io.jroIO_base import *
12 12 from schainpy.utils import log
13 13
14 14
15 15 class HDFReader(Reader, ProcessingUnit):
16 16 """Processing unit to read HDF5 format files
17 17
18 18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 20 attributes.
21 21 It is possible to read any HDF5 file by given the structure in the `description`
22 22 parameter, also you can add extra values to metadata with the parameter `extras`.
23 23
24 24 Parameters:
25 25 -----------
26 26 path : str
27 27 Path where files are located.
28 28 startDate : date
29 29 Start date of the files
30 30 endDate : list
31 31 End date of the files
32 32 startTime : time
33 33 Start time of the files
34 34 endTime : time
35 35 End time of the files
36 36 description : dict, optional
37 37 Dictionary with the description of the HDF5 file
38 38 extras : dict, optional
39 39 Dictionary with extra metadata to be be added to `dataOut`
40 40
41 41 Examples
42 42 --------
43 43
44 44 desc = {
45 45 'Data': {
46 46 'data_output': ['u', 'v', 'w'],
47 47 'utctime': 'timestamps',
48 48 } ,
49 49 'Metadata': {
50 50 'heightList': 'heights'
51 51 }
52 52 }
53 53
54 54 desc = {
55 55 'Data': {
56 56 'data_output': 'winds',
57 57 'utctime': 'timestamps'
58 58 },
59 59 'Metadata': {
60 60 'heightList': 'heights'
61 61 }
62 62 }
63 63
64 64 extras = {
65 65 'timeZone': 300
66 66 }
67 67
68 68 reader = project.addReadUnit(
69 69 name='HDFReader',
70 70 path='/path/to/files',
71 71 startDate='2019/01/01',
72 72 endDate='2019/01/31',
73 73 startTime='00:00:00',
74 74 endTime='23:59:59',
75 75 # description=json.dumps(desc),
76 76 # extras=json.dumps(extras),
77 77 )
78 78
79 79 """
80 80
81 81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82 82
83 83 def __init__(self):
84 84 ProcessingUnit.__init__(self)
85 85
86 86 self.ext = ".hdf5"
87 87 self.optchar = "D"
88 88 self.meta = {}
89 89 self.data = {}
90 90 self.open_file = h5py.File
91 91 self.open_mode = 'r'
92 92 self.description = {}
93 93 self.extras = {}
94 94 self.filefmt = "*%Y%j***"
95 95 self.folderfmt = "*%Y%j"
96 96 self.utcoffset = 0
97 97
98 98 self.dataOut = Parameters()
99 99 self.dataOut.error=False ## NOTE: Importante definir esto antes inicio
100 100 self.dataOut.flagNoData = True
101 101
102 102 def setup(self, **kwargs):
103 103
104 104 self.set_kwargs(**kwargs)
105 105 if not self.ext.startswith('.'):
106 106 self.ext = '.{}'.format(self.ext)
107 107
108 108 if self.online:
109 109 log.log("Searching files in online mode...", self.name)
110 110
111 111 for nTries in range(self.nTries):
112 112 fullpath = self.searchFilesOnLine(self.path, self.startDate,
113 113 self.endDate, self.expLabel, self.ext, self.walk,
114 114 self.filefmt, self.folderfmt)
115 115 pathname, filename = os.path.split(fullpath)
116 116
117 117 try:
118 118 fullpath = next(fullpath)
119 119
120 120 except:
121 121 fullpath = None
122 122
123 123 if fullpath:
124 124 break
125 125
126 126 log.warning(
127 127 'Waiting {} sec for a valid file in {}: try {} ...'.format(
128 128 self.delay, self.path, nTries + 1),
129 129 self.name)
130 130 time.sleep(self.delay)
131 131
132 132 if not(fullpath):
133 133 raise schainpy.admin.SchainError(
134 134 'There isn\'t any valid file in {}'.format(self.path))
135 135
136 136 pathname, filename = os.path.split(fullpath)
137 137 self.year = int(filename[1:5])
138 138 self.doy = int(filename[5:8])
139 139 self.set = int(filename[8:11]) - 1
140 140 else:
141 141 log.log("Searching files in {}".format(self.path), self.name)
142 142 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
143 143 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
144 144
145 145 self.setNextFile()
146 146
147 147
148 148
149 149
150 150 def readFirstHeader(self):
151 151 '''Read metadata and data'''
152 152
153 153 self.__readMetadata()
154 154 self.__readData()
155 155 self.__setBlockList()
156 156
157 157 for attr in self.meta:
158 158 setattr(self.dataOut, attr, self.meta[attr])
159 159 self.blockIndex = 0
160 160
161 161 return
162 162
163 163 def __setBlockList(self):
164 164 '''
165 165 Selects the data within the times defined
166 166
167 167 self.fp
168 168 self.startTime
169 169 self.endTime
170 170 self.blockList
171 171 self.blocksPerFile
172 172
173 173 '''
174 174
175 175 startTime = self.startTime
176 176 endTime = self.endTime
177 177 thisUtcTime = self.data['utctime'] + self.utcoffset
178 178 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
179 179 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
180 180 self.startFileDatetime = thisDatetime
181 181 thisDate = thisDatetime.date()
182 182 thisTime = thisDatetime.time()
183 183
184 184 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
185 185 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
186 186
187 187 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
188 188
189 189 self.blockList = ind
190 190 self.blocksPerFile = len(ind)
191 191 self.blocksPerFile = len(thisUtcTime)
192 192 return
193 193
194 194 def __readMetadata(self):
195 195 '''
196 196 Reads Metadata
197 197 '''
198 198
199 199 meta = {}
200 200
201 201 if self.description:
202 202 for key, value in self.description['Metadata'].items():
203 203 meta[key] = self.fp[value][()]
204 204 else:
205 205 grp = self.fp['Metadata']
206 206 for name in grp:
207 207 meta[name] = grp[name][()]
208 208
209 209 if self.extras:
210 210 for key, value in self.extras.items():
211 211 meta[key] = value
212 212 self.meta = meta
213 213
214 214 return
215 215
216 216
217 217
218 218 def checkForRealPath(self, nextFile, nextDay):
219 219
220 220 # print("check FRP")
221 221 # dt = self.startFileDatetime + datetime.timedelta(1)
222 222 # filename = '{}.{}{}'.format(self.path, dt.strftime('%Y%m%d'), self.ext)
223 223 # fullfilename = os.path.join(self.path, filename)
224 224 # print("check Path ",fullfilename,filename)
225 225 # if os.path.exists(fullfilename):
226 226 # return fullfilename, filename
227 227 # return None, filename
228 228 return None,None
229 229
230 230 def __readData(self):
231 231
232 232 data = {}
233 233
234 234 if self.description:
235 235 for key, value in self.description['Data'].items():
236 236 if isinstance(value, str):
237 237 if isinstance(self.fp[value], h5py.Dataset):
238 238 data[key] = self.fp[value][()]
239 239 elif isinstance(self.fp[value], h5py.Group):
240 240 array = []
241 241 for ch in self.fp[value]:
242 242 array.append(self.fp[value][ch][()])
243 243 data[key] = numpy.array(array)
244 244 elif isinstance(value, list):
245 245 array = []
246 246 for ch in value:
247 247 array.append(self.fp[ch][()])
248 248 data[key] = numpy.array(array)
249 249 else:
250 250 grp = self.fp['Data']
251 251 for name in grp:
252 252 if isinstance(grp[name], h5py.Dataset):
253 253 array = grp[name][()]
254 254 elif isinstance(grp[name], h5py.Group):
255 255 array = []
256 256 for ch in grp[name]:
257 257 array.append(grp[name][ch][()])
258 258 array = numpy.array(array)
259 259 else:
260 260 log.warning('Unknown type: {}'.format(name))
261 261
262 262 if name in self.description:
263 263 key = self.description[name]
264 264 else:
265 265 key = name
266 266 data[key] = array
267 267
268 268 self.data = data
269 269 return
270 270
271 271 def getData(self):
272 272 if not self.isDateTimeInRange(self.startFileDatetime, self.startDate, self.endDate, self.startTime, self.endTime):
273 273 self.dataOut.flagNoData = True
274 274 self.blockIndex = self.blocksPerFile
275 275 self.dataOut.error = True # TERMINA EL PROGRAMA
276 276 return
277 277 for attr in self.data:
278 278
279 279 if self.data[attr].ndim == 1:
280 280 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
281 281 else:
282 282 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
283 283
284 284
285 285 self.blockIndex += 1
286 286
287 287 if self.blockIndex == 1:
288 288 log.log("Block No. {}/{} -> {}".format(
289 289 self.blockIndex,
290 290 self.blocksPerFile,
291 291 self.dataOut.datatime.ctime()), self.name)
292 292 else:
293 293 log.log("Block No. {}/{} ".format(
294 294 self.blockIndex,
295 295 self.blocksPerFile),self.name)
296 296
297 297 if self.blockIndex == self.blocksPerFile:
298 298 self.setNextFile()
299 299
300 300 self.dataOut.flagNoData = False
301 301
302 302
303 303 def run(self, **kwargs):
304 304
305 305 if not(self.isConfig):
306 306 self.setup(**kwargs)
307 307 self.isConfig = True
308 308
309 309 self.getData()
310 310
311 311 @MPDecorator
312 312 class HDFWriter(Operation):
313 313 """Operation to write HDF5 files.
314 314
315 315 The HDF5 file contains by default two groups Data and Metadata where
316 316 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
317 317 parameters, data attributes are normaly time dependent where the metadata
318 318 are not.
319 319 It is possible to customize the structure of the HDF5 file with the
320 320 optional description parameter see the examples.
321 321
322 322 Parameters:
323 323 -----------
324 324 path : str
325 325 Path where files will be saved.
326 326 blocksPerFile : int
327 327 Number of blocks per file
328 328 metadataList : list
329 329 List of the dataOut attributes that will be saved as metadata
330 330 dataList : int
331 331 List of the dataOut attributes that will be saved as data
332 332 setType : bool
333 333 If True the name of the files corresponds to the timestamp of the data
334 334 description : dict, optional
335 335 Dictionary with the desired description of the HDF5 file
336 336
337 337 Examples
338 338 --------
339 339
340 340 desc = {
341 341 'data_output': {'winds': ['z', 'w', 'v']},
342 342 'utctime': 'timestamps',
343 343 'heightList': 'heights'
344 344 }
345 345 desc = {
346 346 'data_output': ['z', 'w', 'v'],
347 347 'utctime': 'timestamps',
348 348 'heightList': 'heights'
349 349 }
350 350 desc = {
351 351 'Data': {
352 352 'data_output': 'winds',
353 353 'utctime': 'timestamps'
354 354 },
355 355 'Metadata': {
356 356 'heightList': 'heights'
357 357 }
358 358 }
359 359
360 360 writer = proc_unit.addOperation(name='HDFWriter')
361 361 writer.addParameter(name='path', value='/path/to/file')
362 362 writer.addParameter(name='blocksPerFile', value='32')
363 363 writer.addParameter(name='metadataList', value='heightList,timeZone')
364 364 writer.addParameter(name='dataList',value='data_output,utctime')
365 365 # writer.addParameter(name='description',value=json.dumps(desc))
366 366
367 367 """
368 368
369 369 ext = ".hdf5"
370 370 optchar = "D"
371 371 filename = None
372 372 path = None
373 373 setFile = None
374 374 fp = None
375 375 firsttime = True
376 376 #Configurations
377 377 blocksPerFile = None
378 378 blockIndex = None
379 379 dataOut = None #eval ??????
380 380 #Data Arrays
381 381 dataList = None
382 382 metadataList = None
383 383 currentDay = None
384 384 lastTime = None
385 385 timeZone = "ut"
386 386 hourLimit = 3
387 387 breakDays = True
388 388
389 389 def __init__(self):
390 390
391 391 Operation.__init__(self)
392 392
393 393
394 394 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None,
395 395 description={},timeZone = "ut",hourLimit = 3, breakDays=True):
396 396 self.path = path
397 397 self.blocksPerFile = blocksPerFile
398 398 self.metadataList = metadataList
399 399 self.dataList = [s.strip() for s in dataList]
400 400 self.setType = setType
401 401 self.description = description
402 402 self.timeZone = timeZone
403 403 self.hourLimit = hourLimit
404 404 self.breakDays = breakDays
405 405
406 406 if self.metadataList is None:
407 407 self.metadataList = self.dataOut.metadata_list
408 408
409 409 tableList = []
410 410 dsList = []
411 411
412 412 for i in range(len(self.dataList)):
413 413 dsDict = {}
414 414 if hasattr(self.dataOut, self.dataList[i]):
415 415 dataAux = getattr(self.dataOut, self.dataList[i])
416 416 dsDict['variable'] = self.dataList[i]
417 417 else:
418 418 log.warning('Attribute {} not found in dataOut'.format(self.dataList[i]),self.name)
419 419 continue
420 420
421 421 if dataAux is None:
422 422 continue
423 423 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
424 424 dsDict['nDim'] = 0
425 425 else:
426 426 dsDict['nDim'] = len(dataAux.shape)
427 427 dsDict['shape'] = dataAux.shape
428 428 dsDict['dsNumber'] = dataAux.shape[0]
429 429 dsDict['dtype'] = dataAux.dtype
430 430
431 431 dsList.append(dsDict)
432 432
433 433 self.blockIndex = 0
434 434 self.dsList = dsList
435 435 self.currentDay = self.dataOut.datatime.date()
436 436
437 437
438 438 def timeFlag(self):
439 439 currentTime = self.dataOut.utctime
440 440 timeTuple = None
441 441 if self.timeZone == "lt":
442 442 timeTuple = time.localtime(currentTime)
443 443 else :
444 444 timeTuple = time.gmtime(currentTime)
445 445
446 446 dataDay = timeTuple.tm_yday
447 447
448 448 if self.lastTime is None:
449 449 self.lastTime = currentTime
450 450 self.currentDay = dataDay
451 451 return False
452 452
453 453 timeDiff = currentTime - self.lastTime
454 454
455 455 #Si el dia es diferente o si la diferencia entre un dato y otro supera self.hourLimit
456 456 if (dataDay != self.currentDay) and self.breakDays:
457 457 self.currentDay = dataDay
458 458 return True
459 459 elif timeDiff > self.hourLimit*60*60:
460 460 self.lastTime = currentTime
461 461 return True
462 462 else:
463 463 self.lastTime = currentTime
464 464 return False
465 465
466 466 def run(self, dataOut,**kwargs):
467 467
468 self.dataOut = dataOut.copy()
468 self.dataOut = dataOut
469
469 470 if not(self.isConfig):
470 471 self.setup(**kwargs)
471 472
472 473 self.isConfig = True
473 474 self.setNextFile()
474 475
475 476 self.putData()
476 477
477 478 #return self.dataOut
478 479
479 480 def setNextFile(self):
480 481
481 482 ext = self.ext
482 483 path = self.path
483 484 setFile = self.setFile
484 485 timeTuple = None
485 486 if self.timeZone == "lt":
486 487 timeTuple = time.localtime(self.dataOut.utctime)
487 488 elif self.timeZone == "ut":
488 489 timeTuple = time.gmtime(self.dataOut.utctime)
489 490 #print("path: ",timeTuple)
490 491 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
491 492 fullpath = os.path.join(path, subfolder)
492 493
493 494 if os.path.exists(fullpath):
494 495 filesList = os.listdir(fullpath)
495 496 filesList = [k for k in filesList if k.startswith(self.optchar)]
496 497 if len( filesList ) > 0:
497 498 filesList = sorted(filesList, key=str.lower)
498 499 filen = filesList[-1]
499 500 # el filename debera tener el siguiente formato
500 501 # 0 1234 567 89A BCDE (hex)
501 502 # x YYYY DDD SSS .ext
502 503 if isNumber(filen[8:11]):
503 504 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
504 505 else:
505 506 setFile = -1
506 507 else:
507 508 setFile = -1 #inicializo mi contador de seteo
508 509 else:
509 510 os.makedirs(fullpath)
510 511 setFile = -1 #inicializo mi contador de seteo
511 512
512 513 if self.setType is None:
513 514 setFile += 1
514 515 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
515 516 timeTuple.tm_year,
516 517 timeTuple.tm_yday,
517 518 setFile,
518 519 ext )
519 520 else:
520 521 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
521 522 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
522 523 timeTuple.tm_year,
523 524 timeTuple.tm_yday,
524 525 setFile,
525 526 ext )
526 527
527 528 self.filename = os.path.join( path, subfolder, file )
528 529
529 530
530 531
531 532 def getLabel(self, name, x=None):
532 533
533 534 if x is None:
534 535 if 'Data' in self.description:
535 536 data = self.description['Data']
536 537 if 'Metadata' in self.description:
537 538 data.update(self.description['Metadata'])
538 539 else:
539 540 data = self.description
540 541 if name in data:
541 542 if isinstance(data[name], str):
542 543 return data[name]
543 544 elif isinstance(data[name], list):
544 545 return None
545 546 elif isinstance(data[name], dict):
546 547 for key, value in data[name].items():
547 548 return key
548 549 return name
549 550 else:
550 551 if 'Metadata' in self.description:
551 552 meta = self.description['Metadata']
552 553 else:
553 554 meta = self.description
554 555 if name in meta:
555 556 if isinstance(meta[name], list):
556 557 return meta[name][x]
557 558 elif isinstance(meta[name], dict):
558 559 for key, value in meta[name].items():
559 560 return value[x]
560 561 if 'cspc' in name:
561 562 return 'pair{:02d}'.format(x)
562 563 else:
563 564 return 'channel{:02d}'.format(x)
564 565
565 566 def writeMetadata(self, fp):
566 567
567 568 if self.description:
568 569 if 'Metadata' in self.description:
569 570 grp = fp.create_group('Metadata')
570 571 else:
571 572 grp = fp
572 573 else:
573 574 grp = fp.create_group('Metadata')
574 575
575 576 for i in range(len(self.metadataList)):
576 577 if not hasattr(self.dataOut, self.metadataList[i]):
577 578 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
578 579 continue
579 580 value = getattr(self.dataOut, self.metadataList[i])
580 581 if isinstance(value, bool):
581 582 if value is True:
582 583 value = 1
583 584 else:
584 585 value = 0
585 586 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
586 587 return
587 588
588 589 def writeData(self, fp):
589 590
590 591 if self.description:
591 592 if 'Data' in self.description:
592 593 grp = fp.create_group('Data')
593 594 else:
594 595 grp = fp
595 596 else:
596 597 grp = fp.create_group('Data')
597 598
598 599 dtsets = []
599 600 data = []
600 601
601 602 for dsInfo in self.dsList:
602 603 if dsInfo['nDim'] == 0:
603 604 ds = grp.create_dataset(
604 605 self.getLabel(dsInfo['variable']),
605 606 (self.blocksPerFile, ),
606 607 chunks=True,
607 608 dtype=numpy.float64)
608 609 dtsets.append(ds)
609 610 data.append((dsInfo['variable'], -1))
610 611 else:
611 612 label = self.getLabel(dsInfo['variable'])
612 613 if label is not None:
613 614 sgrp = grp.create_group(label)
614 615 else:
615 616 sgrp = grp
616 617 for i in range(dsInfo['dsNumber']):
617 618 ds = sgrp.create_dataset(
618 619 self.getLabel(dsInfo['variable'], i),
619 620 (self.blocksPerFile, ) + dsInfo['shape'][1:],
620 621 chunks=True,
621 622 dtype=dsInfo['dtype'])
622 623 dtsets.append(ds)
623 624 data.append((dsInfo['variable'], i))
624 625 fp.flush()
625 626
626 627 log.log('Creating file: {}'.format(fp.filename), self.name)
627 628
628 629 self.ds = dtsets
629 630 self.data = data
630 631 self.firsttime = True
631 632
632 633 return
633 634
634 635 def putData(self):
635 636
636 637 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
637 638 self.closeFile()
638 639 self.setNextFile()
639 640 self.dataOut.flagNoData = False
640 641 self.blockIndex = 0
641 642 return
642 643
643 644
644 645
645 646 if self.blockIndex == 0:
646 647 #Escribir metadata Aqui???
647 648 #Setting HDF5 File
648 649 self.fp = h5py.File(self.filename, 'w')
649 650 #write metadata
650 651 self.writeMetadata(self.fp)
651 652 #Write data
652 653 self.writeData(self.fp)
653 654 log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
654 655 elif (self.blockIndex % 10 ==0):
655 656 log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
656 657 else:
657 658
658 659 log.log('Block No. {}/{}'.format(self.blockIndex+1, self.blocksPerFile), self.name)
659 660
660 661 for i, ds in enumerate(self.ds):
661 662 attr, ch = self.data[i]
662 663 if ch == -1:
663 664 ds[self.blockIndex] = getattr(self.dataOut, attr)
664 665 else:
665 666 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
666 667
667 668 self.blockIndex += 1
668 669
669 670 self.fp.flush()
670 671 self.dataOut.flagNoData = True
671 672
672 673
673 674 def closeFile(self):
674 675
675 676 if self.blockIndex != self.blocksPerFile:
676 677 for ds in self.ds:
677 678 ds.resize(self.blockIndex, axis=0)
678 679
679 680 if self.fp:
680 681 self.fp.flush()
681 682 self.fp.close()
682 683
683 684 def close(self):
684 685
685 686 self.closeFile()
Show file before | Show file after | Hide comments
@@ -1,233 +1,233
1 1 '''
2 2 Base clases to create Processing units and operations, the MPDecorator
3 3 must be used in plotting and writing operations to allow to run as an
4 4 external process.
5 5 '''
6 6 import os
7 7 import inspect
8 8 import zmq
9 9 import time
10 10 import pickle
11 11 import traceback
12 12 from threading import Thread
13 13 from multiprocessing import Process, Queue
14 14 from schainpy.utils import log
15 15 import copy
16 16 QUEUE_SIZE = int(os.environ.get('QUEUE_MAX_SIZE', '10'))
17 17 class ProcessingUnit(object):
18 18 '''
19 19 Base class to create Signal Chain Units
20 20 '''
21 21
22 22 proc_type = 'processing'
23 23
24 24 def __init__(self):
25 25
26 26 self.dataIn = None
27 27 self.dataOut = None
28 28 self.isConfig = False
29 29 self.operations = []
30 30
31 31 def setInput(self, unit):
32 32
33 33 self.dataIn = unit.dataOut
34 34
35 35
36 36 def getAllowedArgs(self):
37 37 if hasattr(self, '__attrs__'):
38 38 return self.__attrs__
39 39 else:
40 40 return inspect.getargspec(self.run).args
41 41
42 42 def addOperation(self, conf, operation):
43 43 '''
44 44 '''
45 45
46 46 self.operations.append((operation, conf.type, conf.getKwargs()))
47 47
48 48 def getOperationObj(self, objId):
49 49
50 50 if objId not in list(self.operations.keys()):
51 51 return None
52 52
53 53 return self.operations[objId]
54 54
55 55 def call(self, **kwargs):
56 56
57 57 mybool = (self.dataOut.type == 'Voltage') and self.dataOut.useInputBuffer and (not self.dataOut.buffer_empty) #liberar desde buffer
58 58 try:
59 59
60 60
61 61 if mybool:
62 #print("run jeje")
62 #print("run yeah")
63 63 self.run(**kwargs)
64 64 else:
65 65 if self.dataIn is not None and self.dataIn.flagNoData and not self.dataIn.error:
66 66 return self.dataIn.isReady()
67 67 elif self.dataIn is None or not self.dataIn.error: #unidad de lectura o procesamiento regular
68 68 self.run(**kwargs)
69 69 elif self.dataIn.error:
70 70 self.dataOut.error = self.dataIn.error
71 71 self.dataOut.flagNoData = True
72 72 print("exec proc error")
73 73
74 74 except:
75 75
76 76 err = traceback.format_exc()
77 77 if 'SchainWarning' in err:
78 78 log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), self.name)
79 79 elif 'SchainError' in err:
80 80 log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), self.name)
81 81 else:
82 82 log.error(err, self.name)
83 83 self.dataOut.error = True
84 84
85 85
86 86 for op, optype, opkwargs in self.operations:
87 87
88 88 if (optype == 'other' and self.dataOut.isReady()) or mybool:
89 89 try:
90 90 self.dataOut = op.run(self.dataOut, **opkwargs)
91 91 except Exception as e:
92 92 print(e)
93 93 self.dataOut.error = True
94 94 return 'Error'
95 95 elif optype == 'external' and self.dataOut.isReady() :
96 96 op.queue.put(copy.deepcopy(self.dataOut))
97 97 elif optype == 'external' and self.dataOut.error:
98 98 op.queue.put(copy.deepcopy(self.dataOut))
99 99
100 100
101 101 if not self.dataOut.error:
102 102 if self.dataOut.type == 'Voltage':
103 103 if not self.dataOut.buffer_empty : #continue
104 104 return 'no_Read'
105 105 elif self.dataOut.useInputBuffer and (self.dataOut.buffer_empty) and self.dataOut.isReady() :
106 106 return 'new_Read'
107 107 else:
108 108 return True
109 109 else:
110 110 #print("ret True")
111 111 return True
112 112 else:
113 113 return 'Error'
114 114 #return 'Error' if self.dataOut.error else True #self.dataOut.isReady()
115 115
116 116 def setup(self):
117 117
118 118 raise NotImplementedError
119 119
120 120 def run(self):
121 121
122 122 raise NotImplementedError
123 123
124 124 def close(self):
125 125
126 126 return
127 127
128 128
129 129 class Operation(object):
130 130
131 131 '''
132 132 '''
133 133
134 134 proc_type = 'operation'
135 135
136 136 def __init__(self):
137 137
138 138 self.id = None
139 139 self.isConfig = False
140 140
141 141 if not hasattr(self, 'name'):
142 142 self.name = self.__class__.__name__
143 143
144 144 def getAllowedArgs(self):
145 145 if hasattr(self, '__attrs__'):
146 146 return self.__attrs__
147 147 else:
148 148 return inspect.getargspec(self.run).args
149 149
150 150 def setup(self):
151 151
152 152 self.isConfig = True
153 153
154 154 raise NotImplementedError
155 155
156 156 def run(self, dataIn, **kwargs):
157 157 """
158 158 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
159 159 atributos del objeto dataIn.
160 160
161 161 Input:
162 162
163 163 dataIn : objeto del tipo JROData
164 164
165 165 Return:
166 166
167 167 None
168 168
169 169 Affected:
170 170 __buffer : buffer de recepcion de datos.
171 171
172 172 """
173 173 if not self.isConfig:
174 174 self.setup(**kwargs)
175 175
176 176 raise NotImplementedError
177 177
178 178 def close(self):
179 179
180 180 return
181 181
182 182
183 183 def MPDecorator(BaseClass):
184 184 """
185 185 Multiprocessing class decorator
186 186
187 187 This function add multiprocessing features to a BaseClass.
188 188 """
189 189
190 190 class MPClass(BaseClass, Process):
191 191
192 192 def __init__(self, *args, **kwargs):
193 193 super(MPClass, self).__init__()
194 194 Process.__init__(self)
195 195
196 196 self.args = args
197 197 self.kwargs = kwargs
198 198 self.t = time.time()
199 199 self.op_type = 'external'
200 200 self.name = BaseClass.__name__
201 201 self.__doc__ = BaseClass.__doc__
202 202
203 203 if 'plot' in self.name.lower() and not self.name.endswith('_'):
204 204 self.name = '{}{}'.format(self.CODE.upper(), 'Plot')
205 205
206 206 self.start_time = time.time()
207 207 self.err_queue = args[3]
208 208 self.queue = Queue(maxsize=QUEUE_SIZE)
209 209 self.myrun = BaseClass.run
210 210
211 211 def run(self):
212 212
213 while True:
213 while 1:
214 214
215 215 dataOut = self.queue.get()
216 216
217 217 if not dataOut.error:
218 218 try:
219 219 BaseClass.run(self, dataOut, **self.kwargs)
220 220 except:
221 221 err = traceback.format_exc()
222 222 log.error(err, self.name)
223 223 else:
224 224 break
225 225
226 226 self.close()
227 227
228 228 def close(self):
229 229
230 230 BaseClass.close(self)
231 231 log.success('Done...(Time:{:4.2f} secs)'.format(time.time()-self.start_time), self.name)
232 232
233 233 return MPClass
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,1948 +1,2076
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Spectra processing Unit and operations
6 6
7 7 Here you will find the processing unit `SpectraProc` and several operations
8 8 to work with Spectra data type
9 9 """
10 10
11 11 import time
12 12 import itertools
13 13
14 14 import numpy
15 15 import math
16 16
17 17 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
18 18 from schainpy.model.data.jrodata import Spectra
19 19 from schainpy.model.data.jrodata import hildebrand_sekhon
20 20 from schainpy.model.data import _noise
21 21
22 22 from schainpy.utils import log
23 23 import matplotlib.pyplot as plt
24 from scipy.optimize import curve_fit
24 #from scipy.optimize import curve_fit
25 25
26 26 class SpectraProc(ProcessingUnit):
27 27
28 28 def __init__(self):
29 29
30 30 ProcessingUnit.__init__(self)
31 31
32 32 self.buffer = None
33 33 self.firstdatatime = None
34 34 self.profIndex = 0
35 35 self.dataOut = Spectra()
36 36 self.id_min = None
37 37 self.id_max = None
38 38 self.setupReq = False #Agregar a todas las unidades de proc
39 39
40 40 def __updateSpecFromVoltage(self):
41 41
42 42
43 43
44 44 self.dataOut.timeZone = self.dataIn.timeZone
45 45 self.dataOut.dstFlag = self.dataIn.dstFlag
46 46 self.dataOut.errorCount = self.dataIn.errorCount
47 47 self.dataOut.useLocalTime = self.dataIn.useLocalTime
48 48 try:
49 49 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
50 50 except:
51 51 pass
52 52 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
53 53 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
54 54 self.dataOut.channelList = self.dataIn.channelList
55 55 self.dataOut.heightList = self.dataIn.heightList
56 56 self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
57 57 self.dataOut.nProfiles = self.dataOut.nFFTPoints
58 58 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
59 59 self.dataOut.utctime = self.firstdatatime
60 60 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
61 61 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
62 62 self.dataOut.flagShiftFFT = False
63 63 self.dataOut.nCohInt = self.dataIn.nCohInt
64 64 self.dataOut.nIncohInt = 1
65 65 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
66 66 self.dataOut.frequency = self.dataIn.frequency
67 67 self.dataOut.realtime = self.dataIn.realtime
68 68 self.dataOut.azimuth = self.dataIn.azimuth
69 69 self.dataOut.zenith = self.dataIn.zenith
70 70 self.dataOut.codeList = self.dataIn.codeList
71 71 self.dataOut.azimuthList = self.dataIn.azimuthList
72 72 self.dataOut.elevationList = self.dataIn.elevationList
73 73
74 74
75 75 def __getFft(self):
76 76 """
77 77 Convierte valores de Voltaje a Spectra
78 78
79 79 Affected:
80 80 self.dataOut.data_spc
81 81 self.dataOut.data_cspc
82 82 self.dataOut.data_dc
83 83 self.dataOut.heightList
84 84 self.profIndex
85 85 self.buffer
86 86 self.dataOut.flagNoData
87 87 """
88 88 fft_volt = numpy.fft.fft(
89 89 self.buffer, n=self.dataOut.nFFTPoints, axis=1)
90 90 fft_volt = fft_volt.astype(numpy.dtype('complex'))
91 91 dc = fft_volt[:, 0, :]
92 92
93 93 # calculo de self-spectra
94 94 fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
95 95 spc = fft_volt * numpy.conjugate(fft_volt)
96 96 spc = spc.real
97 97
98 98 blocksize = 0
99 99 blocksize += dc.size
100 100 blocksize += spc.size
101 101
102 102 cspc = None
103 103 pairIndex = 0
104 104 if self.dataOut.pairsList != None:
105 105 # calculo de cross-spectra
106 106 cspc = numpy.zeros(
107 107 (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
108 108 for pair in self.dataOut.pairsList:
109 109 if pair[0] not in self.dataOut.channelList:
110 110 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
111 111 str(pair), str(self.dataOut.channelList)))
112 112 if pair[1] not in self.dataOut.channelList:
113 113 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
114 114 str(pair), str(self.dataOut.channelList)))
115 115
116 116 cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
117 117 numpy.conjugate(fft_volt[pair[1], :, :])
118 118 pairIndex += 1
119 119 blocksize += cspc.size
120 120
121 121 self.dataOut.data_spc = spc
122 122 self.dataOut.data_cspc = cspc
123 123 self.dataOut.data_dc = dc
124 124 self.dataOut.blockSize = blocksize
125 125 self.dataOut.flagShiftFFT = False
126 126
127 127 def run(self, nProfiles=None, nFFTPoints=None, pairsList=None, ippFactor=None, shift_fft=False):
128 128 #print("run spc proc")
129 try:
130 type = self.dataIn.type.decode("utf-8")
131 self.dataIn.type = type
132 except:
133 pass
129 134 if self.dataIn.type == "Spectra":
130 135
131 136 try:
132 137 self.dataOut.copy(self.dataIn)
133 138
134 139 except Exception as e:
135 140 print("Error dataIn ",e)
136 141
137 142 if shift_fft:
138 143 #desplaza a la derecha en el eje 2 determinadas posiciones
139 144 shift = int(self.dataOut.nFFTPoints/2)
140 145 self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
141 146
142 147 if self.dataOut.data_cspc is not None:
143 148 #desplaza a la derecha en el eje 2 determinadas posiciones
144 149 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
145 150 if pairsList:
146 151 self.__selectPairs(pairsList)
147 152
148 153
149 154 elif self.dataIn.type == "Voltage":
150 155
151 156 self.dataOut.flagNoData = True
152 157
153 158 if nFFTPoints == None:
154 159 raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
155 160
156 161 if nProfiles == None:
157 162 nProfiles = nFFTPoints
158 163
159 164 if ippFactor == None:
160 165 self.dataOut.ippFactor = 1
161 166
162 167 self.dataOut.nFFTPoints = nFFTPoints
163 168 #print(" volts ch,prof, h: ", self.dataIn.data.shape)
164 169 if self.buffer is None:
165 170 self.buffer = numpy.zeros((self.dataIn.nChannels,
166 171 nProfiles,
167 172 self.dataIn.nHeights),
168 173 dtype='complex')
169 174
170 175 if self.dataIn.flagDataAsBlock:
171 176 nVoltProfiles = self.dataIn.data.shape[1]
172 177
173 178 if nVoltProfiles == nProfiles:
174 179 self.buffer = self.dataIn.data.copy()
175 180 self.profIndex = nVoltProfiles
176 181
177 182 elif nVoltProfiles < nProfiles:
178 183
179 184 if self.profIndex == 0:
180 185 self.id_min = 0
181 186 self.id_max = nVoltProfiles
182 187
183 188 self.buffer[:, self.id_min:self.id_max,
184 189 :] = self.dataIn.data
185 190 self.profIndex += nVoltProfiles
186 191 self.id_min += nVoltProfiles
187 192 self.id_max += nVoltProfiles
188 193 else:
189 194 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
190 195 self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
191 196 self.dataOut.flagNoData = True
192 197 else:
193 198 self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
194 199 self.profIndex += 1
195 200
196 201 if self.firstdatatime == None:
197 202 self.firstdatatime = self.dataIn.utctime
198 203
199 204 if self.profIndex == nProfiles:
200 205
201 206 self.__updateSpecFromVoltage()
202 207
203 208 if pairsList == None:
204 209 self.dataOut.pairsList = [pair for pair in itertools.combinations(self.dataOut.channelList, 2)]
205 210 else:
206 211 self.dataOut.pairsList = pairsList
207 212 self.__getFft()
208 213 self.dataOut.flagNoData = False
209 214 self.firstdatatime = None
210 215 self.profIndex = 0
211 216
217 elif self.dataIn.type == "Parameters":
218
219 self.dataOut.data_spc = self.dataIn.data_spc
220 self.dataOut.data_cspc = self.dataIn.data_cspc
221 self.dataOut.data_outlier = self.dataIn.data_outlier
222 self.dataOut.nProfiles = self.dataIn.nProfiles
223 self.dataOut.nIncohInt = self.dataIn.nIncohInt
224 self.dataOut.nFFTPoints = self.dataIn.nFFTPoints
225 self.dataOut.ippFactor = self.dataIn.ippFactor
226 self.dataOut.max_nIncohInt = self.dataIn.max_nIncohInt
227 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
228 self.dataOut.ipp = self.dataIn.ipp
229 #self.dataOut.abscissaList = self.dataIn.getVelRange(1)
230 #self.dataOut.spc_noise = self.dataIn.getNoise()
231 #self.dataOut.spc_range = (self.dataIn.getFreqRange(1) , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1))
232 # self.dataOut.normFactor = self.dataIn.normFactor
233 if hasattr(self.dataIn, 'channelList'):
234 self.dataOut.channelList = self.dataIn.channelList
235 if hasattr(self.dataIn, 'pairsList'):
236 self.dataOut.pairsList = self.dataIn.pairsList
237 self.dataOut.groupList = self.dataIn.pairsList
238
239 self.dataOut.flagNoData = False
240
241 if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels
242 self.dataOut.ChanDist = self.dataIn.ChanDist
243 else: self.dataOut.ChanDist = None
244
245 #if hasattr(self.dataIn, 'VelRange'): #Velocities range
246 # self.dataOut.VelRange = self.dataIn.VelRange
247 #else: self.dataOut.VelRange = None
248
249
212 250
213 251 else:
214 raise ValueError("The type of input object '%s' is not valid".format(
252 raise ValueError("The type of input object {} is not valid".format(
215 253 self.dataIn.type))
216 254
217 255
218 256 def __selectPairs(self, pairsList):
219 257
220 258 if not pairsList:
221 259 return
222 260
223 261 pairs = []
224 262 pairsIndex = []
225 263
226 264 for pair in pairsList:
227 265 if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
228 266 continue
229 267 pairs.append(pair)
230 268 pairsIndex.append(pairs.index(pair))
231 269
232 270 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
233 271 self.dataOut.pairsList = pairs
234 272
235 273 return
236 274
237 275 def selectFFTs(self, minFFT, maxFFT ):
238 276 """
239 277 Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
240 278 minFFT<= FFT <= maxFFT
241 279 """
242 280
243 281 if (minFFT > maxFFT):
244 282 raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT))
245 283
246 284 if (minFFT < self.dataOut.getFreqRange()[0]):
247 285 minFFT = self.dataOut.getFreqRange()[0]
248 286
249 287 if (maxFFT > self.dataOut.getFreqRange()[-1]):
250 288 maxFFT = self.dataOut.getFreqRange()[-1]
251 289
252 290 minIndex = 0
253 291 maxIndex = 0
254 292 FFTs = self.dataOut.getFreqRange()
255 293
256 294 inda = numpy.where(FFTs >= minFFT)
257 295 indb = numpy.where(FFTs <= maxFFT)
258 296
259 297 try:
260 298 minIndex = inda[0][0]
261 299 except:
262 300 minIndex = 0
263 301
264 302 try:
265 303 maxIndex = indb[0][-1]
266 304 except:
267 305 maxIndex = len(FFTs)
268 306
269 307 self.selectFFTsByIndex(minIndex, maxIndex)
270 308
271 309 return 1
272 310
273 311 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
274 312 newheis = numpy.where(
275 313 self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
276 314
277 315 if hei_ref != None:
278 316 newheis = numpy.where(self.dataOut.heightList > hei_ref)
279 317
280 318 minIndex = min(newheis[0])
281 319 maxIndex = max(newheis[0])
282 320 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
283 321 heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
284 322
285 323 # determina indices
286 324 nheis = int(self.dataOut.radarControllerHeaderObj.txB /
287 325 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
288 326 avg_dB = 10 * \
289 327 numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
290 328 beacon_dB = numpy.sort(avg_dB)[-nheis:]
291 329 beacon_heiIndexList = []
292 330 for val in avg_dB.tolist():
293 331 if val >= beacon_dB[0]:
294 332 beacon_heiIndexList.append(avg_dB.tolist().index(val))
295 333
296 334 #data_spc = data_spc[:,:,beacon_heiIndexList]
297 335 data_cspc = None
298 336 if self.dataOut.data_cspc is not None:
299 337 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
300 338 #data_cspc = data_cspc[:,:,beacon_heiIndexList]
301 339
302 340 data_dc = None
303 341 if self.dataOut.data_dc is not None:
304 342 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
305 343 #data_dc = data_dc[:,beacon_heiIndexList]
306 344
307 345 self.dataOut.data_spc = data_spc
308 346 self.dataOut.data_cspc = data_cspc
309 347 self.dataOut.data_dc = data_dc
310 348 self.dataOut.heightList = heightList
311 349 self.dataOut.beacon_heiIndexList = beacon_heiIndexList
312 350
313 351 return 1
314 352
315 353 def selectFFTsByIndex(self, minIndex, maxIndex):
316 354 """
317 355
318 356 """
319 357
320 358 if (minIndex < 0) or (minIndex > maxIndex):
321 359 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (minIndex, maxIndex))
322 360
323 361 if (maxIndex >= self.dataOut.nProfiles):
324 362 maxIndex = self.dataOut.nProfiles-1
325 363
326 364 #Spectra
327 365 data_spc = self.dataOut.data_spc[:,minIndex:maxIndex+1,:]
328 366
329 367 data_cspc = None
330 368 if self.dataOut.data_cspc is not None:
331 369 data_cspc = self.dataOut.data_cspc[:,minIndex:maxIndex+1,:]
332 370
333 371 data_dc = None
334 372 if self.dataOut.data_dc is not None:
335 373 data_dc = self.dataOut.data_dc[minIndex:maxIndex+1,:]
336 374
337 375 self.dataOut.data_spc = data_spc
338 376 self.dataOut.data_cspc = data_cspc
339 377 self.dataOut.data_dc = data_dc
340 378
341 379 self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1])
342 380 self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1]
343 381 self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1]
344 382
345 383 return 1
346 384
347 385 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
348 386 # validacion de rango
349 387 if minHei == None:
350 388 minHei = self.dataOut.heightList[0]
351 389
352 390 if maxHei == None:
353 391 maxHei = self.dataOut.heightList[-1]
354 392
355 393 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
356 394 print('minHei: %.2f is out of the heights range' % (minHei))
357 395 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
358 396 minHei = self.dataOut.heightList[0]
359 397
360 398 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
361 399 print('maxHei: %.2f is out of the heights range' % (maxHei))
362 400 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
363 401 maxHei = self.dataOut.heightList[-1]
364 402
365 403 # validacion de velocidades
366 404 velrange = self.dataOut.getVelRange(1)
367 405
368 406 if minVel == None:
369 407 minVel = velrange[0]
370 408
371 409 if maxVel == None:
372 410 maxVel = velrange[-1]
373 411
374 412 if (minVel < velrange[0]) or (minVel > maxVel):
375 413 print('minVel: %.2f is out of the velocity range' % (minVel))
376 414 print('minVel is setting to %.2f' % (velrange[0]))
377 415 minVel = velrange[0]
378 416
379 417 if (maxVel > velrange[-1]) or (maxVel < minVel):
380 418 print('maxVel: %.2f is out of the velocity range' % (maxVel))
381 419 print('maxVel is setting to %.2f' % (velrange[-1]))
382 420 maxVel = velrange[-1]
383 421
384 422 # seleccion de indices para rango
385 423 minIndex = 0
386 424 maxIndex = 0
387 425 heights = self.dataOut.heightList
388 426
389 427 inda = numpy.where(heights >= minHei)
390 428 indb = numpy.where(heights <= maxHei)
391 429
392 430 try:
393 431 minIndex = inda[0][0]
394 432 except:
395 433 minIndex = 0
396 434
397 435 try:
398 436 maxIndex = indb[0][-1]
399 437 except:
400 438 maxIndex = len(heights)
401 439
402 440 if (minIndex < 0) or (minIndex > maxIndex):
403 441 raise ValueError("some value in (%d,%d) is not valid" % (
404 442 minIndex, maxIndex))
405 443
406 444 if (maxIndex >= self.dataOut.nHeights):
407 445 maxIndex = self.dataOut.nHeights - 1
408 446
409 447 # seleccion de indices para velocidades
410 448 indminvel = numpy.where(velrange >= minVel)
411 449 indmaxvel = numpy.where(velrange <= maxVel)
412 450 try:
413 451 minIndexVel = indminvel[0][0]
414 452 except:
415 453 minIndexVel = 0
416 454
417 455 try:
418 456 maxIndexVel = indmaxvel[0][-1]
419 457 except:
420 458 maxIndexVel = len(velrange)
421 459
422 460 # seleccion del espectro
423 461 data_spc = self.dataOut.data_spc[:,
424 462 minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
425 463 # estimacion de ruido
426 464 noise = numpy.zeros(self.dataOut.nChannels)
427 465
428 466 for channel in range(self.dataOut.nChannels):
429 467 daux = data_spc[channel, :, :]
430 468 sortdata = numpy.sort(daux, axis=None)
431 469 noise[channel] = hildebrand_sekhon(sortdata, self.dataOut.nIncohInt)
432 470
433 471 self.dataOut.noise_estimation = noise.copy()
434 472
435 473 return 1
436 474
437 475 class removeDC(Operation):
438 476
439 477 def run(self, dataOut, mode=2):
440 478 self.dataOut = dataOut
441 479 jspectra = self.dataOut.data_spc
442 480 jcspectra = self.dataOut.data_cspc
443 481
444 482 num_chan = jspectra.shape[0]
445 483 num_hei = jspectra.shape[2]
446 484
447 485 if jcspectra is not None:
448 486 jcspectraExist = True
449 487 num_pairs = jcspectra.shape[0]
450 488 else:
451 489 jcspectraExist = False
452 490
453 491 freq_dc = int(jspectra.shape[1] / 2)
454 492 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
455 493 ind_vel = ind_vel.astype(int)
456 494
457 495 if ind_vel[0] < 0:
458 496 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
459 497
460 498 if mode == 1:
461 499 jspectra[:, freq_dc, :] = (
462 500 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
463 501
464 502 if jcspectraExist:
465 503 jcspectra[:, freq_dc, :] = (
466 504 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
467 505
468 506 if mode == 2:
469 507
470 508 vel = numpy.array([-2, -1, 1, 2])
471 509 xx = numpy.zeros([4, 4])
472 510
473 511 for fil in range(4):
474 512 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
475 513
476 514 xx_inv = numpy.linalg.inv(xx)
477 515 xx_aux = xx_inv[0, :]
478 516
479 517 for ich in range(num_chan):
480 518 yy = jspectra[ich, ind_vel, :]
481 519 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
482 520
483 521 junkid = jspectra[ich, freq_dc, :] <= 0
484 522 cjunkid = sum(junkid)
485 523
486 524 if cjunkid.any():
487 525 jspectra[ich, freq_dc, junkid.nonzero()] = (
488 526 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
489 527
490 528 if jcspectraExist:
491 529 for ip in range(num_pairs):
492 530 yy = jcspectra[ip, ind_vel, :]
493 531 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
494 532
495 533 self.dataOut.data_spc = jspectra
496 534 self.dataOut.data_cspc = jcspectra
497 535
498 536 return self.dataOut
499 537
500 class getNoise(Operation):
501 warnings = False
538 class getNoiseB(Operation):
539
540 __slots__ =('offset','warnings', 'isConfig', 'minIndex','maxIndex','minIndexFFT','maxIndexFFT')
502 541 def __init__(self):
503 542
504 543 Operation.__init__(self)
544 self.isConfig = False
545
546 def setup(self, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
505 547
506 def run(self, dataOut, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
507 self.dataOut = dataOut
508 548 self.warnings = warnings
509 549 if minHei == None:
510 550 minHei = self.dataOut.heightList[0]
511 551
512 552 if maxHei == None:
513 553 maxHei = self.dataOut.heightList[-1]
514 554
515 555 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
516 556 if self.warnings:
517 557 print('minHei: %.2f is out of the heights range' % (minHei))
518 558 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
519 559 minHei = self.dataOut.heightList[0]
520 560
521 561 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
522 562 if self.warnings:
523 563 print('maxHei: %.2f is out of the heights range' % (maxHei))
524 564 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
525 565 maxHei = self.dataOut.heightList[-1]
526 566
527 567
528 568 #indices relativos a los puntos de fft, puede ser de acuerdo a velocidad o frecuencia
529 569 minIndexFFT = 0
530 570 maxIndexFFT = 0
531 571 # validacion de velocidades
532 572 indminPoint = None
533 573 indmaxPoint = None
534 574 if self.dataOut.type == 'Spectra':
535 575 if minVel == None and maxVel == None :
536 576
537 577 freqrange = self.dataOut.getFreqRange(1)
538 578
539 579 if minFreq == None:
540 580 minFreq = freqrange[0]
541 581
542 582 if maxFreq == None:
543 583 maxFreq = freqrange[-1]
544 584
545 585 if (minFreq < freqrange[0]) or (minFreq > maxFreq):
546 586 if self.warnings:
547 587 print('minFreq: %.2f is out of the frequency range' % (minFreq))
548 588 print('minFreq is setting to %.2f' % (freqrange[0]))
549 589 minFreq = freqrange[0]
550 590
551 591 if (maxFreq > freqrange[-1]) or (maxFreq < minFreq):
552 592 if self.warnings:
553 593 print('maxFreq: %.2f is out of the frequency range' % (maxFreq))
554 594 print('maxFreq is setting to %.2f' % (freqrange[-1]))
555 595 maxFreq = freqrange[-1]
556 596
557 597 indminPoint = numpy.where(freqrange >= minFreq)
558 598 indmaxPoint = numpy.where(freqrange <= maxFreq)
559 599
560 600 else:
561 601
562 602 velrange = self.dataOut.getVelRange(1)
563 603
564 604 if minVel == None:
565 605 minVel = velrange[0]
566 606
567 607 if maxVel == None:
568 608 maxVel = velrange[-1]
569 609
570 610 if (minVel < velrange[0]) or (minVel > maxVel):
571 611 if self.warnings:
572 612 print('minVel: %.2f is out of the velocity range' % (minVel))
573 613 print('minVel is setting to %.2f' % (velrange[0]))
574 614 minVel = velrange[0]
575 615
576 616 if (maxVel > velrange[-1]) or (maxVel < minVel):
577 617 if self.warnings:
578 618 print('maxVel: %.2f is out of the velocity range' % (maxVel))
579 619 print('maxVel is setting to %.2f' % (velrange[-1]))
580 620 maxVel = velrange[-1]
581 621
582 622 indminPoint = numpy.where(velrange >= minVel)
583 623 indmaxPoint = numpy.where(velrange <= maxVel)
584 624
585 625
586 626 # seleccion de indices para rango
587 627 minIndex = 0
588 628 maxIndex = 0
589 629 heights = self.dataOut.heightList
590 630
591 631 inda = numpy.where(heights >= minHei)
592 632 indb = numpy.where(heights <= maxHei)
593 633
594 634 try:
595 635 minIndex = inda[0][0]
596 636 except:
597 637 minIndex = 0
598 638
599 639 try:
600 640 maxIndex = indb[0][-1]
601 641 except:
602 642 maxIndex = len(heights)
603 643
604 644 if (minIndex < 0) or (minIndex > maxIndex):
605 645 raise ValueError("some value in (%d,%d) is not valid" % (
606 646 minIndex, maxIndex))
607 647
608 648 if (maxIndex >= self.dataOut.nHeights):
609 649 maxIndex = self.dataOut.nHeights - 1
610 650 #############################################################3
611 651 # seleccion de indices para velocidades
612 652 if self.dataOut.type == 'Spectra':
613 653 try:
614 654 minIndexFFT = indminPoint[0][0]
615 655 except:
616 656 minIndexFFT = 0
617 657
618 658 try:
619 659 maxIndexFFT = indmaxPoint[0][-1]
620 660 except:
621 661 maxIndexFFT = len( self.dataOut.getFreqRange(1))
622 662
663 self.minIndex, self.maxIndex, self.minIndexFFT, self.maxIndexFFT = minIndex, maxIndex, minIndexFFT, maxIndexFFT
664 self.isConfig = True
665 if offset!=None:
666 self.offset = 10**(offset/10)
667 #print("config getNoise Done")
668
669 def run(self, dataOut, offset=None, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
670 self.dataOut = dataOut
671
672 if not self.isConfig:
673 self.setup(offset, minHei, maxHei,minVel, maxVel, minFreq, maxFreq, warnings)
623 674
624 675 self.dataOut.noise_estimation = None
625 676 noise = None
626 677 if self.dataOut.type == 'Voltage':
627 noise = self.dataOut.getNoise(ymin_index=minIndex, ymax_index=maxIndex)
678 noise = self.dataOut.getNoise(ymin_index=self.minIndex, ymax_index=self.maxIndex)
628 679 #print(minIndex, maxIndex,minIndexVel, maxIndexVel)
629 680 elif self.dataOut.type == 'Spectra':
630 noise = self.dataOut.getNoise(xmin_index=minIndexFFT, xmax_index=maxIndexFFT, ymin_index=minIndex, ymax_index=maxIndex)
681
682 noise = numpy.zeros( self.dataOut.nChannels)
683 for channel in range( self.dataOut.nChannels):
684 norm = self.dataOut.max_nIncohInt/self.dataOut.nIncohInt[channel, self.minIndex:self.maxIndex]
685 #print("norm nIncoh: ", norm )
686 daux = self.dataOut.data_spc[channel,self.minIndexFFT:self.maxIndexFFT, self.minIndex:self.maxIndex]
687 daux = numpy.multiply(daux, norm)
688 #print("offset: ", self.offset, 10*numpy.log10(self.offset))
689 #noise[channel] = self.getNoiseByMean(daux)/self.offset
690 noise[channel] = self.getNoiseByHS(daux, self.dataOut.max_nIncohInt)/self.offset
691
692 #noise = self.dataOut.getNoise(xmin_index=self.minIndexFFT, xmax_index=self.maxIndexFFT, ymin_index=self.minIndex, ymax_index=self.maxIndex)
631 693 else:
632 noise = self.dataOut.getNoise(xmin_index=minIndexFFT, xmax_index=maxIndexFFT, ymin_index=minIndex, ymax_index=maxIndex)
694 noise = self.dataOut.getNoise(xmin_index=self.minIndexFFT, xmax_index=self.maxIndexFFT, ymin_index=self.minIndex, ymax_index=self.maxIndex)
633 695 self.dataOut.noise_estimation = noise.copy() # dataOut.noise
634 696 #print("2: ",10*numpy.log10(self.dataOut.noise_estimation/64))
635 697
698 #print(self.dataOut.flagNoData)
636 699 return self.dataOut
637 700
701 def getNoiseByMean(self,data):
702 #data debe estar ordenado
703 data = numpy.mean(data,axis=1)
704 sortdata = numpy.sort(data, axis=None)
705 #sortID=data.argsort()
706 #print(data.shape)
707
708 pnoise = None
709 j = 0
710
711 mean = numpy.mean(sortdata)
712 min = numpy.min(sortdata)
713 delta = mean - min
714 indexes = numpy.where(sortdata > (mean+delta))[0] #only array of indexes
715 #print(len(indexes))
716 if len(indexes)==0:
717 pnoise = numpy.mean(sortdata)
718 else:
719 j = indexes[0]
720 pnoise = numpy.mean(sortdata[0:j])
721
722 # from matplotlib import pyplot as plt
723 # plt.plot(sortdata)
724 # plt.vlines(j,(pnoise-delta),(pnoise+delta), color='r')
725 # plt.show()
726 #print("noise: ", 10*numpy.log10(pnoise))
727 return pnoise
728
729 def getNoiseByHS(self,data, navg):
730 #data debe estar ordenado
731 #data = numpy.mean(data,axis=1)
732 sortdata = numpy.sort(data, axis=None)
733
734 lenOfData = len(sortdata)
735 nums_min = lenOfData*0.05
736
737 if nums_min <= 5:
738
739 nums_min = 5
740
741 sump = 0.
742 sumq = 0.
743
744 j = 0
745 cont = 1
746
747 while((cont == 1)and(j < lenOfData)):
748
749 sump += sortdata[j]
750 sumq += sortdata[j]**2
751 #sumq -= sump**2
752 if j > nums_min:
753 rtest = float(j)/(j-1) + 1.0/0.1
754 #if ((sumq*j) > (sump**2)):
755 if ((sumq*j) > (rtest*sump**2)):
756 j = j - 1
757 sump = sump - sortdata[j]
758 sumq = sumq - sortdata[j]**2
759 cont = 0
760
761 j += 1
762
763 lnoise = sump / j
764
765 return lnoise
638 766
639 767
640 # import matplotlib.pyplot as plt
641 768
642 769 def fit_func( x, a0, a1, a2): #, a3, a4, a5):
643 770 z = (x - a1) / a2
644 771 y = a0 * numpy.exp(-z**2 / a2) #+ a3 + a4 * x + a5 * x**2
645 772 return y
646 773
647 774
648 775 class CleanRayleigh(Operation):
649 776
650 777 def __init__(self):
651 778
652 779 Operation.__init__(self)
653 780 self.i=0
654 781 self.isConfig = False
655 782 self.__dataReady = False
656 783 self.__profIndex = 0
657 784 self.byTime = False
658 785 self.byProfiles = False
659 786
660 787 self.bloques = None
661 788 self.bloque0 = None
662 789
663 790 self.index = 0
664 791
665 792 self.buffer = 0
666 793 self.buffer2 = 0
667 794 self.buffer3 = 0
668 795
669 796
670 797 def setup(self,dataOut,min_hei,max_hei,n, timeInterval,factor_stdv):
671 798
672 799 self.nChannels = dataOut.nChannels
673 800 self.nProf = dataOut.nProfiles
674 801 self.nPairs = dataOut.data_cspc.shape[0]
675 802 self.pairsArray = numpy.array(dataOut.pairsList)
676 803 self.spectra = dataOut.data_spc
677 804 self.cspectra = dataOut.data_cspc
678 805 self.heights = dataOut.heightList #alturas totales
679 806 self.nHeights = len(self.heights)
680 807 self.min_hei = min_hei
681 808 self.max_hei = max_hei
682 809 if (self.min_hei == None):
683 810 self.min_hei = 0
684 811 if (self.max_hei == None):
685 812 self.max_hei = dataOut.heightList[-1]
686 813 self.hval = ((self.max_hei>=self.heights) & (self.heights >= self.min_hei)).nonzero()
687 814 self.heightsClean = self.heights[self.hval] #alturas filtradas
688 815 self.hval = self.hval[0] # forma (N,), an solo N elementos -> Indices de alturas
689 816 self.nHeightsClean = len(self.heightsClean)
690 817 self.channels = dataOut.channelList
691 818 self.nChan = len(self.channels)
692 819 self.nIncohInt = dataOut.nIncohInt
693 820 self.__initime = dataOut.utctime
694 821 self.maxAltInd = self.hval[-1]+1
695 822 self.minAltInd = self.hval[0]
696 823
697 824 self.crosspairs = dataOut.pairsList
698 825 self.nPairs = len(self.crosspairs)
699 826 self.normFactor = dataOut.normFactor
700 827 self.nFFTPoints = dataOut.nFFTPoints
701 828 self.ippSeconds = dataOut.ippSeconds
702 829 self.currentTime = self.__initime
703 830 self.pairsArray = numpy.array(dataOut.pairsList)
704 831 self.factor_stdv = factor_stdv
705 832
706 833 if n != None :
707 834 self.byProfiles = True
708 835 self.nIntProfiles = n
709 836 else:
710 837 self.__integrationtime = timeInterval
711 838
712 839 self.__dataReady = False
713 840 self.isConfig = True
714 841
715 842
716 843
717 844 def run(self, dataOut,min_hei=None,max_hei=None, n=None, timeInterval=10,factor_stdv=2.5):
718 845 #print("runing cleanRayleigh")
719 846 if not self.isConfig :
720 847
721 848 self.setup(dataOut, min_hei,max_hei,n,timeInterval,factor_stdv)
722 849
723 850 tini=dataOut.utctime
724 851
725 852 if self.byProfiles:
726 853 if self.__profIndex == self.nIntProfiles:
727 854 self.__dataReady = True
728 855 else:
729 856 if (tini - self.__initime) >= self.__integrationtime:
730 857
731 858 self.__dataReady = True
732 859 self.__initime = tini
733 860
734 861 #if (tini.tm_min % 2) == 0 and (tini.tm_sec < 5 and self.fint==0):
735 862
736 863 if self.__dataReady:
737 864
738 865 self.__profIndex = 0
739 866 jspc = self.buffer
740 867 jcspc = self.buffer2
741 868 #jnoise = self.buffer3
742 869 self.buffer = dataOut.data_spc
743 870 self.buffer2 = dataOut.data_cspc
744 871 #self.buffer3 = dataOut.noise
745 872 self.currentTime = dataOut.utctime
746 873 if numpy.any(jspc) :
747 874 #print( jspc.shape, jcspc.shape)
748 875 jspc = numpy.reshape(jspc,(int(len(jspc)/self.nChannels),self.nChannels,self.nFFTPoints,self.nHeights))
749 876 try:
750 877 jcspc= numpy.reshape(jcspc,(int(len(jcspc)/self.nPairs),self.nPairs,self.nFFTPoints,self.nHeights))
751 878 except:
752 879 print("no cspc")
753 880 self.__dataReady = False
754 881 #print( jspc.shape, jcspc.shape)
755 882 dataOut.flagNoData = False
756 883 else:
757 884 dataOut.flagNoData = True
758 885 self.__dataReady = False
759 886 return dataOut
760 887 else:
761 888 #print( len(self.buffer))
762 889 if numpy.any(self.buffer):
763 890 self.buffer = numpy.concatenate((self.buffer,dataOut.data_spc), axis=0)
764 891 try:
765 892 self.buffer2 = numpy.concatenate((self.buffer2,dataOut.data_cspc), axis=0)
766 893 self.buffer3 += dataOut.data_dc
767 894 except:
768 895 pass
769 896 else:
770 897 self.buffer = dataOut.data_spc
771 898 self.buffer2 = dataOut.data_cspc
772 899 self.buffer3 = dataOut.data_dc
773 900 #print self.index, self.fint
774 901 #print self.buffer2.shape
775 902 dataOut.flagNoData = True ## NOTE: ?? revisar LUEGO
776 903 self.__profIndex += 1
777 904 return dataOut ## NOTE: REV
778 905
779 906
780 907 #index = tini.tm_hour*12+tini.tm_min/5
781 908 '''
782 REVISAR
909 #REVISAR
783 910 '''
784 911 # jspc = jspc/self.nFFTPoints/self.normFactor
785 912 # jcspc = jcspc/self.nFFTPoints/self.normFactor
786 913
787 914
788 915
789 916 tmp_spectra,tmp_cspectra = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
790 917 dataOut.data_spc = tmp_spectra
791 918 dataOut.data_cspc = tmp_cspectra
792 919
793 920 #dataOut.data_spc,dataOut.data_cspc = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
794 921
795 922 dataOut.data_dc = self.buffer3
796 923 dataOut.nIncohInt *= self.nIntProfiles
797 924 dataOut.max_nIncohInt = self.nIntProfiles
798 925 dataOut.utctime = self.currentTime #tiempo promediado
799 926 #print("Time: ",time.localtime(dataOut.utctime))
800 927 # dataOut.data_spc = sat_spectra
801 928 # dataOut.data_cspc = sat_cspectra
802 929 self.buffer = 0
803 930 self.buffer2 = 0
804 931 self.buffer3 = 0
805 932
806 933 return dataOut
807 934
808 935 def cleanRayleigh(self,dataOut,spectra,cspectra,factor_stdv):
809 936 print("OP cleanRayleigh")
810 937 #import matplotlib.pyplot as plt
811 938 #for k in range(149):
812 939 #channelsProcssd = []
813 940 #channelA_ok = False
814 941 #rfunc = cspectra.copy() #self.bloques
815 942 rfunc = spectra.copy()
816 943 #rfunc = cspectra
817 944 #val_spc = spectra*0.0 #self.bloque0*0.0
818 945 #val_cspc = cspectra*0.0 #self.bloques*0.0
819 946 #in_sat_spectra = spectra.copy() #self.bloque0
820 947 #in_sat_cspectra = cspectra.copy() #self.bloques
821 948
822 949
823 950 ###ONLY FOR TEST:
824 951 raxs = math.ceil(math.sqrt(self.nPairs))
825 952 if raxs == 0:
826 953 raxs = 1
827 954 caxs = math.ceil(self.nPairs/raxs)
828 955 if self.nPairs <4:
829 956 raxs = 2
830 957 caxs = 2
831 958 #print(raxs, caxs)
832 959 fft_rev = 14 #nFFT to plot
833 960 hei_rev = ((self.heights >= 550) & (self.heights <= 551)).nonzero() #hei to plot
834 961 hei_rev = hei_rev[0]
835 962 #print(hei_rev)
836 963
837 964 #print numpy.absolute(rfunc[:,0,0,14])
838 965
839 966 gauss_fit, covariance = None, None
840 967 for ih in range(self.minAltInd,self.maxAltInd):
841 968 for ifreq in range(self.nFFTPoints):
842 969 '''
843 970 ###ONLY FOR TEST:
844 971 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
845 972 fig, axs = plt.subplots(raxs, caxs)
846 973 fig2, axs2 = plt.subplots(raxs, caxs)
847 974 col_ax = 0
848 975 row_ax = 0
849 976 '''
850 977 #print(self.nPairs)
851 978 for ii in range(self.nChan): #PARES DE CANALES SELF y CROSS
852 979 # if self.crosspairs[ii][1]-self.crosspairs[ii][0] > 1: # APLICAR SOLO EN PARES CONTIGUOS
853 980 # continue
854 981 # if not self.crosspairs[ii][0] in channelsProcssd:
855 982 # channelA_ok = True
856 983 #print("pair: ",self.crosspairs[ii])
857 984 '''
858 985 ###ONLY FOR TEST:
859 986 if (col_ax%caxs==0 and col_ax!=0 and self.nPairs !=1):
860 987 col_ax = 0
861 988 row_ax += 1
862 989 '''
863 990 func2clean = 10*numpy.log10(numpy.absolute(rfunc[:,ii,ifreq,ih])) #Potencia?
864 991 #print(func2clean.shape)
865 992 val = (numpy.isfinite(func2clean)==True).nonzero()
866 993
867 994 if len(val)>0: #limitador
868 995 min_val = numpy.around(numpy.amin(func2clean)-2) #> (-40)
869 996 if min_val <= -40 :
870 997 min_val = -40
871 998 max_val = numpy.around(numpy.amax(func2clean)+2) #< 200
872 999 if max_val >= 200 :
873 1000 max_val = 200
874 1001 #print min_val, max_val
875 1002 step = 1
876 1003 #print("Getting bins and the histogram")
877 1004 x_dist = min_val + numpy.arange(1 + ((max_val-(min_val))/step))*step
878 1005 y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
879 1006 #print(len(y_dist),len(binstep[:-1]))
880 1007 #print(row_ax,col_ax, " ..")
881 1008 #print(self.pairsArray[ii][0],self.pairsArray[ii][1])
882 1009 mean = numpy.sum(x_dist * y_dist) / numpy.sum(y_dist)
883 1010 sigma = numpy.sqrt(numpy.sum(y_dist * (x_dist - mean)**2) / numpy.sum(y_dist))
884 1011 parg = [numpy.amax(y_dist),mean,sigma]
885 1012
886 1013 newY = None
887 1014
888 1015 try :
889 1016 gauss_fit, covariance = curve_fit(fit_func, x_dist, y_dist,p0=parg)
890 1017 mode = gauss_fit[1]
891 1018 stdv = gauss_fit[2]
892 1019 #print(" FIT OK",gauss_fit)
893 1020 '''
894 1021 ###ONLY FOR TEST:
895 1022 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
896 1023 newY = fit_func(x_dist,gauss_fit[0],gauss_fit[1],gauss_fit[2])
897 1024 axs[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
898 1025 axs[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
899 1026 axs[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
900 1027 '''
901 1028 except:
902 1029 mode = mean
903 1030 stdv = sigma
904 1031 #print("FIT FAIL")
905 1032 #continue
906 1033
907 1034
908 1035 #print(mode,stdv)
909 1036 #Removing echoes greater than mode + std_factor*stdv
910 1037 noval = (abs(func2clean - mode)>=(factor_stdv*stdv)).nonzero()
911 1038 #noval tiene los indices que se van a remover
912 1039 #print("Chan ",ii," novals: ",len(noval[0]))
913 1040 if len(noval[0]) > 0: #forma de array (N,) es igual a longitud (N)
914 1041 novall = ((func2clean - mode) >= (factor_stdv*stdv)).nonzero()
915 1042 #print(novall)
916 1043 #print(" ",self.pairsArray[ii])
917 1044 #cross_pairs = self.pairsArray[ii]
918 1045 #Getting coherent echoes which are removed.
919 1046 # if len(novall[0]) > 0:
920 1047 #
921 1048 # val_spc[novall[0],cross_pairs[0],ifreq,ih] = 1
922 1049 # val_spc[novall[0],cross_pairs[1],ifreq,ih] = 1
923 1050 # val_cspc[novall[0],ii,ifreq,ih] = 1
924 1051 #print("OUT NOVALL 1")
925 1052 try:
926 1053 pair = (self.channels[ii],self.channels[ii + 1])
927 1054 except:
928 1055 pair = (99,99)
929 1056 #print("par ", pair)
930 1057 if ( pair in self.crosspairs):
931 1058 q = self.crosspairs.index(pair)
932 1059 #print("está aqui: ", q, (ii,ii + 1))
933 1060 new_a = numpy.delete(cspectra[:,q,ifreq,ih], noval[0])
934 1061 cspectra[noval,q,ifreq,ih] = numpy.mean(new_a) #mean CrossSpectra
935 1062
936 1063 #if channelA_ok:
937 1064 #chA = self.channels.index(cross_pairs[0])
938 1065 new_b = numpy.delete(spectra[:,ii,ifreq,ih], noval[0])
939 1066 spectra[noval,ii,ifreq,ih] = numpy.mean(new_b) #mean Spectra Pair A
940 1067 #channelA_ok = False
941 1068
942 1069 # chB = self.channels.index(cross_pairs[1])
943 1070 # new_c = numpy.delete(spectra[:,chB,ifreq,ih], noval[0])
944 1071 # spectra[noval,chB,ifreq,ih] = numpy.mean(new_c) #mean Spectra Pair B
945 1072 #
946 1073 # channelsProcssd.append(self.crosspairs[ii][0]) # save channel A
947 1074 # channelsProcssd.append(self.crosspairs[ii][1]) # save channel B
948 1075 '''
949 1076 ###ONLY FOR TEST:
950 1077 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
951 1078 func2clean = 10*numpy.log10(numpy.absolute(spectra[:,ii,ifreq,ih]))
952 1079 y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
953 1080 axs2[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
954 1081 axs2[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
955 1082 axs2[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
956 1083 '''
957 1084 '''
958 1085 ###ONLY FOR TEST:
959 1086 col_ax += 1 #contador de ploteo columnas
960 1087 ##print(col_ax)
961 1088 ###ONLY FOR TEST:
962 1089 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
963 1090 title = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km"
964 1091 title2 = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km CLEANED"
965 1092 fig.suptitle(title)
966 1093 fig2.suptitle(title2)
967 1094 plt.show()
968 1095 '''
969 1096 ##################################################################################################
970 1097
971 1098 #print("Getting average of the spectra and cross-spectra from incoherent echoes.")
972 1099 out_spectra = numpy.zeros([self.nChan,self.nFFTPoints,self.nHeights], dtype=float) #+numpy.nan
973 1100 out_cspectra = numpy.zeros([self.nPairs,self.nFFTPoints,self.nHeights], dtype=complex) #+numpy.nan
974 1101 for ih in range(self.nHeights):
975 1102 for ifreq in range(self.nFFTPoints):
976 1103 for ich in range(self.nChan):
977 1104 tmp = spectra[:,ich,ifreq,ih]
978 1105 valid = (numpy.isfinite(tmp[:])==True).nonzero()
979 1106
980 1107 if len(valid[0]) >0 :
981 1108 out_spectra[ich,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
982 1109
983 1110 for icr in range(self.nPairs):
984 1111 tmp = numpy.squeeze(cspectra[:,icr,ifreq,ih])
985 1112 valid = (numpy.isfinite(tmp)==True).nonzero()
986 1113 if len(valid[0]) > 0:
987 1114 out_cspectra[icr,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
988 1115
989 1116 return out_spectra, out_cspectra
990 1117
991 1118 def REM_ISOLATED_POINTS(self,array,rth):
992 1119 # import matplotlib.pyplot as plt
993 1120 if rth == None :
994 1121 rth = 4
995 1122 #print("REM ISO")
996 1123 num_prof = len(array[0,:,0])
997 1124 num_hei = len(array[0,0,:])
998 1125 n2d = len(array[:,0,0])
999 1126
1000 1127 for ii in range(n2d) :
1001 1128 #print ii,n2d
1002 1129 tmp = array[ii,:,:]
1003 1130 #print tmp.shape, array[ii,101,:],array[ii,102,:]
1004 1131
1005 1132 # fig = plt.figure(figsize=(6,5))
1006 1133 # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
1007 1134 # ax = fig.add_axes([left, bottom, width, height])
1008 1135 # x = range(num_prof)
1009 1136 # y = range(num_hei)
1010 1137 # cp = ax.contour(y,x,tmp)
1011 1138 # ax.clabel(cp, inline=True,fontsize=10)
1012 1139 # plt.show()
1013 1140
1014 1141 #indxs = WHERE(FINITE(tmp) AND tmp GT 0,cindxs)
1015 1142 tmp = numpy.reshape(tmp,num_prof*num_hei)
1016 1143 indxs1 = (numpy.isfinite(tmp)==True).nonzero()
1017 1144 indxs2 = (tmp > 0).nonzero()
1018 1145
1019 1146 indxs1 = (indxs1[0])
1020 1147 indxs2 = indxs2[0]
1021 1148 #indxs1 = numpy.array(indxs1[0])
1022 1149 #indxs2 = numpy.array(indxs2[0])
1023 1150 indxs = None
1024 1151 #print indxs1 , indxs2
1025 1152 for iv in range(len(indxs2)):
1026 1153 indv = numpy.array((indxs1 == indxs2[iv]).nonzero())
1027 1154 #print len(indxs2), indv
1028 1155 if len(indv[0]) > 0 :
1029 1156 indxs = numpy.concatenate((indxs,indxs2[iv]), axis=None)
1030 1157 # print indxs
1031 1158 indxs = indxs[1:]
1032 1159 #print(indxs, len(indxs))
1033 1160 if len(indxs) < 4 :
1034 1161 array[ii,:,:] = 0.
1035 1162 return
1036 1163
1037 1164 xpos = numpy.mod(indxs ,num_hei)
1038 1165 ypos = (indxs / num_hei)
1039 1166 sx = numpy.argsort(xpos) # Ordering respect to "x" (time)
1040 1167 #print sx
1041 1168 xpos = xpos[sx]
1042 1169 ypos = ypos[sx]
1043 1170
1044 1171 # *********************************** Cleaning isolated points **********************************
1045 1172 ic = 0
1046 1173 while True :
1047 1174 r = numpy.sqrt(list(numpy.power((xpos[ic]-xpos),2)+ numpy.power((ypos[ic]-ypos),2)))
1048 1175 #no_coh = WHERE(FINITE(r) AND (r LE rth),cno_coh)
1049 1176 #plt.plot(r)
1050 1177 #plt.show()
1051 1178 no_coh1 = (numpy.isfinite(r)==True).nonzero()
1052 1179 no_coh2 = (r <= rth).nonzero()
1053 1180 #print r, no_coh1, no_coh2
1054 1181 no_coh1 = numpy.array(no_coh1[0])
1055 1182 no_coh2 = numpy.array(no_coh2[0])
1056 1183 no_coh = None
1057 1184 #print valid1 , valid2
1058 1185 for iv in range(len(no_coh2)):
1059 1186 indv = numpy.array((no_coh1 == no_coh2[iv]).nonzero())
1060 1187 if len(indv[0]) > 0 :
1061 1188 no_coh = numpy.concatenate((no_coh,no_coh2[iv]), axis=None)
1062 1189 no_coh = no_coh[1:]
1063 1190 #print len(no_coh), no_coh
1064 1191 if len(no_coh) < 4 :
1065 1192 #print xpos[ic], ypos[ic], ic
1066 1193 # plt.plot(r)
1067 1194 # plt.show()
1068 1195 xpos[ic] = numpy.nan
1069 1196 ypos[ic] = numpy.nan
1070 1197
1071 1198 ic = ic + 1
1072 1199 if (ic == len(indxs)) :
1073 1200 break
1074 1201 #print( xpos, ypos)
1075 1202
1076 1203 indxs = (numpy.isfinite(list(xpos))==True).nonzero()
1077 1204 #print indxs[0]
1078 1205 if len(indxs[0]) < 4 :
1079 1206 array[ii,:,:] = 0.
1080 1207 return
1081 1208
1082 1209 xpos = xpos[indxs[0]]
1083 1210 ypos = ypos[indxs[0]]
1084 1211 for i in range(0,len(ypos)):
1085 1212 ypos[i]=int(ypos[i])
1086 1213 junk = tmp
1087 1214 tmp = junk*0.0
1088 1215
1089 1216 tmp[list(xpos + (ypos*num_hei))] = junk[list(xpos + (ypos*num_hei))]
1090 1217 array[ii,:,:] = numpy.reshape(tmp,(num_prof,num_hei))
1091 1218
1092 1219 #print array.shape
1093 1220 #tmp = numpy.reshape(tmp,(num_prof,num_hei))
1094 1221 #print tmp.shape
1095 1222
1096 1223 # fig = plt.figure(figsize=(6,5))
1097 1224 # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
1098 1225 # ax = fig.add_axes([left, bottom, width, height])
1099 1226 # x = range(num_prof)
1100 1227 # y = range(num_hei)
1101 1228 # cp = ax.contour(y,x,array[ii,:,:])
1102 1229 # ax.clabel(cp, inline=True,fontsize=10)
1103 1230 # plt.show()
1104 1231 return array
1105 1232
1106 1233
1107 1234 class IntegrationFaradaySpectra(Operation):
1108 1235
1109 1236 __profIndex = 0
1110 1237 __withOverapping = False
1111 1238
1112 1239 __byTime = False
1113 1240 __initime = None
1114 1241 __lastdatatime = None
1115 1242 __integrationtime = None
1116 1243
1117 1244 __buffer_spc = None
1118 1245 __buffer_cspc = None
1119 1246 __buffer_dc = None
1120 1247
1121 1248 __dataReady = False
1122 1249
1123 1250 __timeInterval = None
1124 1251 n_ints = None #matriz de numero de integracions (CH,HEI)
1125 1252 n = None
1126 1253 minHei_ind = None
1127 1254 maxHei_ind = None
1128 1255 navg = 1.0
1129 1256 factor = 0.0
1130 1257 dataoutliers = None # (CHANNELS, HEIGHTS)
1131 1258
1132 1259 def __init__(self):
1133 1260
1134 1261 Operation.__init__(self)
1135 1262
1136 1263 def setup(self, dataOut,n=None, timeInterval=None, overlapping=False, DPL=None, minHei=None, maxHei=None, avg=1,factor=0.75):
1137 1264 """
1138 1265 Set the parameters of the integration class.
1139 1266
1140 1267 Inputs:
1141 1268
1142 1269 n : Number of coherent integrations
1143 1270 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1144 1271 overlapping :
1145 1272
1146 1273 """
1147 1274
1148 1275 self.__initime = None
1149 1276 self.__lastdatatime = 0
1150 1277
1151 1278 self.__buffer_spc = []
1152 1279 self.__buffer_cspc = []
1153 1280 self.__buffer_dc = 0
1154 1281
1155 1282 self.__profIndex = 0
1156 1283 self.__dataReady = False
1157 1284 self.__byTime = False
1158 1285
1159 1286 self.factor = factor
1160 1287 self.navg = avg
1161 1288 #self.ByLags = dataOut.ByLags ###REDEFINIR
1162 1289 self.ByLags = False
1163 1290 self.maxProfilesInt = 1
1164 1291
1165 1292 if DPL != None:
1166 1293 self.DPL=DPL
1167 1294 else:
1168 1295 #self.DPL=dataOut.DPL ###REDEFINIR
1169 1296 self.DPL=0
1170 1297
1171 1298 if n is None and timeInterval is None:
1172 1299 raise ValueError("n or timeInterval should be specified ...")
1173 1300
1174 1301 if n is not None:
1175 1302 self.n = int(n)
1176 1303 else:
1177 1304 self.__integrationtime = int(timeInterval)
1178 1305 self.n = None
1179 1306 self.__byTime = True
1180 1307
1181 1308 if minHei == None:
1182 1309 minHei = self.dataOut.heightList[0]
1183 1310
1184 1311 if maxHei == None:
1185 1312 maxHei = self.dataOut.heightList[-1]
1186 1313
1187 1314 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
1188 1315 print('minHei: %.2f is out of the heights range' % (minHei))
1189 1316 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
1190 1317 minHei = self.dataOut.heightList[0]
1191 1318
1192 1319 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
1193 1320 print('maxHei: %.2f is out of the heights range' % (maxHei))
1194 1321 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
1195 1322 maxHei = self.dataOut.heightList[-1]
1196 1323
1197 1324 ind_list1 = numpy.where(self.dataOut.heightList >= minHei)
1198 1325 ind_list2 = numpy.where(self.dataOut.heightList <= maxHei)
1199 1326 self.minHei_ind = ind_list1[0][0]
1200 1327 self.maxHei_ind = ind_list2[0][-1]
1201 1328
1202 1329 def putData(self, data_spc, data_cspc, data_dc):
1203 1330 """
1204 1331 Add a profile to the __buffer_spc and increase in one the __profileIndex
1205 1332
1206 1333 """
1207 1334
1208 1335 self.__buffer_spc.append(data_spc)
1209 1336
1210 1337 if data_cspc is None:
1211 1338 self.__buffer_cspc = None
1212 1339 else:
1213 1340 self.__buffer_cspc.append(data_cspc)
1214 1341
1215 1342 if data_dc is None:
1216 1343 self.__buffer_dc = None
1217 1344 else:
1218 1345 self.__buffer_dc += data_dc
1219 1346
1220 1347 self.__profIndex += 1
1221 1348
1222 1349 return
1223 1350
1224 1351 def hildebrand_sekhon_Integration(self,sortdata,navg, factor):
1225 1352 #data debe estar ordenado
1226 1353 #sortdata = numpy.sort(data, axis=None)
1227 1354 #sortID=data.argsort()
1228 1355 lenOfData = len(sortdata)
1229 1356 nums_min = lenOfData*factor
1230 1357 if nums_min <= 5:
1231 1358 nums_min = 5
1232 1359 sump = 0.
1233 1360 sumq = 0.
1234 1361 j = 0
1235 1362 cont = 1
1236 1363 while((cont == 1)and(j < lenOfData)):
1237 1364 sump += sortdata[j]
1238 1365 sumq += sortdata[j]**2
1239 1366 if j > nums_min:
1240 1367 rtest = float(j)/(j-1) + 1.0/navg
1241 1368 if ((sumq*j) > (rtest*sump**2)):
1242 1369 j = j - 1
1243 1370 sump = sump - sortdata[j]
1244 1371 sumq = sumq - sortdata[j]**2
1245 1372 cont = 0
1246 1373 j += 1
1247 1374 #lnoise = sump / j
1248 1375 #print("H S done")
1249 1376 #return j,sortID
1250 1377 return j
1251 1378
1252 1379
1253 1380 def pushData(self):
1254 1381 """
1255 1382 Return the sum of the last profiles and the profiles used in the sum.
1256 1383
1257 1384 Affected:
1258 1385
1259 1386 self.__profileIndex
1260 1387
1261 1388 """
1262 1389 bufferH=None
1263 1390 buffer=None
1264 1391 buffer1=None
1265 1392 buffer_cspc=None
1266 1393 self.__buffer_spc=numpy.array(self.__buffer_spc)
1267 1394 try:
1268 1395 self.__buffer_cspc=numpy.array(self.__buffer_cspc)
1269 1396 except :
1270 1397 #print("No cpsc",e)
1271 1398 pass
1272 1399 #print("FREQ_DC",self.__buffer_spc.shape,self.__buffer_cspc.shape)
1273 1400
1274 1401 freq_dc = int(self.__buffer_spc.shape[2] / 2)
1275 1402 #print("FREQ_DC",freq_dc,self.__buffer_spc.shape,self.nHeights)
1276 1403
1277 1404 self.dataOutliers = numpy.zeros((self.nChannels,self.nHeights)) # --> almacen de outliers
1278 1405
1279 1406 for k in range(self.minHei_ind,self.maxHei_ind):
1280 1407 try:
1281 1408 buffer_cspc=numpy.copy(self.__buffer_cspc[:,:,:,k])
1282 1409 except:
1283 1410 #print("No cpsc",e)
1284 1411 pass
1285 1412 outliers_IDs_cspc=[]
1286 1413 cspc_outliers_exist=False
1287 1414 for i in range(self.nChannels):#dataOut.nChannels):
1288 1415
1289 1416 buffer1=numpy.copy(self.__buffer_spc[:,i,:,k])
1290 1417 indexes=[]
1291 1418 #sortIDs=[]
1292 1419 outliers_IDs=[]
1293 1420
1294 1421 for j in range(self.nProfiles): #frecuencias en el tiempo
1295 1422 # if i==0 and j==freq_dc: #NOT CONSIDERING DC PROFILE AT CHANNEL 0
1296 1423 # continue
1297 1424 # if i==1 and j==0: #NOT CONSIDERING DC PROFILE AT CHANNEL 1
1298 1425 # continue
1299 1426 buffer=buffer1[:,j]
1300 1427 sortdata = numpy.sort(buffer, axis=None)
1301 1428
1302 1429 sortID=buffer.argsort()
1303 1430 index = _noise.hildebrand_sekhon2(sortdata,self.navg)
1304 1431
1305 1432 #index,sortID=self.hildebrand_sekhon_Integration(buffer,1,self.factor)
1306 1433
1307 1434 # fig,ax = plt.subplots()
1308 1435 # ax.set_title(str(k)+" "+str(j))
1309 1436 # x=range(len(sortdata))
1310 1437 # ax.scatter(x,sortdata)
1311 1438 # ax.axvline(index)
1312 1439 # plt.show()
1313 1440
1314 1441 indexes.append(index)
1315 1442 #sortIDs.append(sortID)
1316 1443 outliers_IDs=numpy.append(outliers_IDs,sortID[index:])
1317 1444
1318 1445 #print("Outliers: ",outliers_IDs)
1319 1446 outliers_IDs=numpy.array(outliers_IDs)
1320 1447 outliers_IDs=outliers_IDs.ravel()
1321 1448 outliers_IDs=numpy.unique(outliers_IDs)
1322 1449 outliers_IDs=outliers_IDs.astype(numpy.dtype('int64'))
1323 1450 indexes=numpy.array(indexes)
1324 1451 indexmin=numpy.min(indexes)
1325 1452
1326 1453
1327 1454 #print(indexmin,buffer1.shape[0], k)
1328 1455
1329 1456 # fig,ax = plt.subplots()
1330 1457 # ax.plot(sortdata)
1331 1458 # ax2 = ax.twinx()
1332 1459 # x=range(len(indexes))
1333 1460 # #plt.scatter(x,indexes)
1334 1461 # ax2.scatter(x,indexes)
1335 1462 # plt.show()
1336 1463
1337 1464 if indexmin != buffer1.shape[0]:
1338 1465 if self.nChannels > 1:
1339 1466 cspc_outliers_exist= True
1340 1467
1341 1468 lt=outliers_IDs
1342 1469 #avg=numpy.mean(buffer1[[t for t in range(buffer1.shape[0]) if t not in lt],:],axis=0)
1343 1470
1344 1471 for p in list(outliers_IDs):
1345 1472 #buffer1[p,:]=avg
1346 1473 buffer1[p,:] = numpy.NaN
1347 1474
1348 1475 self.dataOutliers[i,k] = len(outliers_IDs)
1349 1476
1350 1477 self.__buffer_spc[:,i,:,k]=numpy.copy(buffer1)
1351 1478
1352 1479 outliers_IDs_cspc=numpy.append(outliers_IDs_cspc,outliers_IDs)
1353 1480
1354 1481
1355 1482
1356 1483 outliers_IDs_cspc=outliers_IDs_cspc.astype(numpy.dtype('int64'))
1357 1484 if cspc_outliers_exist :
1358 1485
1359 1486 lt=outliers_IDs_cspc
1360 1487
1361 1488 #avg=numpy.mean(buffer_cspc[[t for t in range(buffer_cspc.shape[0]) if t not in lt],:],axis=0)
1362 1489 for p in list(outliers_IDs_cspc):
1363 1490 #buffer_cspc[p,:]=avg
1364 1491 buffer_cspc[p,:] = numpy.NaN
1365 1492
1366 1493 try:
1367 1494 self.__buffer_cspc[:,:,:,k]=numpy.copy(buffer_cspc)
1368 1495 except:
1369 1496 #print("No cpsc",e)
1370 1497 pass
1371 1498 #else:
1372 1499 #break
1373 1500
1374 1501
1375 1502
1376 1503 nOutliers = len(outliers_IDs)
1377 1504 #print("Outliers n: ",self.dataOutliers,nOutliers)
1378 1505 buffer=None
1379 1506 bufferH=None
1380 1507 buffer1=None
1381 1508 buffer_cspc=None
1382 1509
1383 1510
1384 1511 buffer=None
1385 1512
1386 1513 #data_spc = numpy.sum(self.__buffer_spc,axis=0)
1387 1514 data_spc = numpy.nansum(self.__buffer_spc,axis=0)
1388 1515 try:
1389 1516 #data_cspc = numpy.sum(self.__buffer_cspc,axis=0)
1390 1517 data_cspc = numpy.nansum(self.__buffer_cspc,axis=0)
1391 1518 except:
1392 1519 #print("No cpsc",e)
1393 1520 pass
1394 1521
1395 1522
1396 1523 data_dc = self.__buffer_dc
1397 1524 #(CH, HEIGH)
1398 1525 self.maxProfilesInt = self.__profIndex
1399 n = self.__profIndex - self.dataOutliers
1526 n = self.__profIndex - self.dataOutliers # n becomes a matrix
1400 1527
1401 1528 self.__buffer_spc = []
1402 1529 self.__buffer_cspc = []
1403 1530 self.__buffer_dc = 0
1404 1531 self.__profIndex = 0
1405 1532
1406 1533 return data_spc, data_cspc, data_dc, n
1407 1534
1408 1535 def byProfiles(self, *args):
1409 1536
1410 1537 self.__dataReady = False
1411 1538 avgdata_spc = None
1412 1539 avgdata_cspc = None
1413 1540 avgdata_dc = None
1414 1541
1415 1542 self.putData(*args)
1416 1543
1417 1544 if self.__profIndex == self.n:
1418 1545
1419 1546 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1420 1547 self.n_ints = n
1421 1548 self.__dataReady = True
1422 1549
1423 1550 return avgdata_spc, avgdata_cspc, avgdata_dc
1424 1551
1425 1552 def byTime(self, datatime, *args):
1426 1553
1427 1554 self.__dataReady = False
1428 1555 avgdata_spc = None
1429 1556 avgdata_cspc = None
1430 1557 avgdata_dc = None
1431 1558
1432 1559 self.putData(*args)
1433 1560
1434 1561 if (datatime - self.__initime) >= self.__integrationtime:
1435 1562 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1436 1563 self.n_ints = n
1437 1564 self.__dataReady = True
1438 1565
1439 1566 return avgdata_spc, avgdata_cspc, avgdata_dc
1440 1567
1441 1568 def integrate(self, datatime, *args):
1442 1569
1443 1570 if self.__profIndex == 0:
1444 1571 self.__initime = datatime
1445 1572
1446 1573 if self.__byTime:
1447 1574 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
1448 1575 datatime, *args)
1449 1576 else:
1450 1577 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1451 1578
1452 1579 if not self.__dataReady:
1453 1580 return None, None, None, None
1454 1581
1455 1582 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
1456 1583
1457 1584 def run(self, dataOut, n=None, DPL = None,timeInterval=None, overlapping=False, minHei=None, maxHei=None, avg=1, factor=0.75):
1458 self.dataOut = dataOut.copy()
1585 self.dataOut = dataOut
1459 1586 if n == 1:
1460 1587 return self.dataOut
1461 1588
1462 self.dataOut.flagNoData = True
1589
1463 1590 if self.dataOut.nChannels == 1:
1464 1591 self.dataOut.data_cspc = None #si es un solo canal no vale la pena acumular DATOS
1465 1592 #print(self.dataOut.data_spc.shape, self.dataOut.data_cspc)
1466 1593 if not self.isConfig:
1467 1594 self.setup(self.dataOut, n, timeInterval, overlapping,DPL ,minHei, maxHei, avg, factor)
1468 1595 self.isConfig = True
1469 1596
1470 1597 if not self.ByLags:
1471 1598 self.nProfiles=self.dataOut.nProfiles
1472 1599 self.nChannels=self.dataOut.nChannels
1473 1600 self.nHeights=self.dataOut.nHeights
1474 1601 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
1475 1602 self.dataOut.data_spc,
1476 1603 self.dataOut.data_cspc,
1477 1604 self.dataOut.data_dc)
1478 1605 else:
1479 1606 self.nProfiles=self.dataOut.nProfiles
1480 1607 self.nChannels=self.dataOut.nChannels
1481 1608 self.nHeights=self.dataOut.nHeights
1482 1609 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
1483 1610 self.dataOut.dataLag_spc,
1484 1611 self.dataOut.dataLag_cspc,
1485 1612 self.dataOut.dataLag_dc)
1486
1613 self.dataOut.flagNoData = True
1487 1614 if self.__dataReady:
1488 1615
1489 1616 if not self.ByLags:
1490 1617 if self.nChannels == 1:
1491 1618 #print("f int", avgdata_spc.shape)
1492 1619 self.dataOut.data_spc = avgdata_spc
1493 1620 self.dataOut.data_cspc = avgdata_spc
1494 1621 else:
1495 1622 self.dataOut.data_spc = numpy.squeeze(avgdata_spc)
1496 1623 self.dataOut.data_cspc = numpy.squeeze(avgdata_cspc)
1497 1624 self.dataOut.data_dc = avgdata_dc
1498 1625 self.dataOut.data_outlier = self.dataOutliers
1499 1626
1500 1627 else:
1501 1628 self.dataOut.dataLag_spc = avgdata_spc
1502 1629 self.dataOut.dataLag_cspc = avgdata_cspc
1503 1630 self.dataOut.dataLag_dc = avgdata_dc
1504 1631
1505 1632 self.dataOut.data_spc=self.dataOut.dataLag_spc[:,:,:,self.dataOut.LagPlot]
1506 1633 self.dataOut.data_cspc=self.dataOut.dataLag_cspc[:,:,:,self.dataOut.LagPlot]
1507 1634 self.dataOut.data_dc=self.dataOut.dataLag_dc[:,:,self.dataOut.LagPlot]
1508 1635
1509 1636
1510 1637 self.dataOut.nIncohInt *= self.n_ints
1511 1638 self.dataOut.max_nIncohInt = self.maxProfilesInt
1512 1639 #print(self.dataOut.max_nIncohInt)
1513 1640 self.dataOut.utctime = avgdatatime
1514 1641 self.dataOut.flagNoData = False
1515 1642 #print("Faraday Integration DONE...")
1643 #print(self.dataOut.flagNoData)
1516 1644 return self.dataOut
1517 1645
1518 1646 class removeInterference(Operation):
1519 1647
1520 1648 def removeInterference2(self):
1521 1649
1522 1650 cspc = self.dataOut.data_cspc
1523 1651 spc = self.dataOut.data_spc
1524 1652 Heights = numpy.arange(cspc.shape[2])
1525 1653 realCspc = numpy.abs(cspc)
1526 1654
1527 1655 for i in range(cspc.shape[0]):
1528 1656 LinePower= numpy.sum(realCspc[i], axis=0)
1529 1657 Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
1530 1658 SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ]
1531 1659 InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
1532 1660 InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
1533 1661 InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
1534 1662
1535 1663
1536 1664 InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
1537 1665 #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
1538 1666 if len(InterferenceRange)<int(cspc.shape[1]*0.3):
1539 1667 cspc[i,InterferenceRange,:] = numpy.NaN
1540 1668
1541 1669 self.dataOut.data_cspc = cspc
1542 1670
1543 1671 def removeInterference(self, interf = 2, hei_interf = None, nhei_interf = None, offhei_interf = None):
1544 1672
1545 1673 jspectra = self.dataOut.data_spc
1546 1674 jcspectra = self.dataOut.data_cspc
1547 1675 jnoise = self.dataOut.getNoise()
1548 1676 num_incoh = self.dataOut.nIncohInt
1549 1677
1550 1678 num_channel = jspectra.shape[0]
1551 1679 num_prof = jspectra.shape[1]
1552 1680 num_hei = jspectra.shape[2]
1553 1681
1554 1682 # hei_interf
1555 1683 if hei_interf is None:
1556 1684 count_hei = int(num_hei / 2)
1557 1685 hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
1558 1686 hei_interf = numpy.asarray(hei_interf)[0]
1559 1687 # nhei_interf
1560 1688 if (nhei_interf == None):
1561 1689 nhei_interf = 5
1562 1690 if (nhei_interf < 1):
1563 1691 nhei_interf = 1
1564 1692 if (nhei_interf > count_hei):
1565 1693 nhei_interf = count_hei
1566 1694 if (offhei_interf == None):
1567 1695 offhei_interf = 0
1568 1696
1569 1697 ind_hei = list(range(num_hei))
1570 1698 # mask_prof = numpy.asarray(range(num_prof - 2)) + 1
1571 1699 # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
1572 1700 mask_prof = numpy.asarray(list(range(num_prof)))
1573 1701 num_mask_prof = mask_prof.size
1574 1702 comp_mask_prof = [0, num_prof / 2]
1575 1703
1576 1704 # noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
1577 1705 if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
1578 1706 jnoise = numpy.nan
1579 1707 noise_exist = jnoise[0] < numpy.Inf
1580 1708
1581 1709 # Subrutina de Remocion de la Interferencia
1582 1710 for ich in range(num_channel):
1583 1711 # Se ordena los espectros segun su potencia (menor a mayor)
1584 1712 power = jspectra[ich, mask_prof, :]
1585 1713 power = power[:, hei_interf]
1586 1714 power = power.sum(axis=0)
1587 1715 psort = power.ravel().argsort()
1588 1716
1589 1717 # Se estima la interferencia promedio en los Espectros de Potencia empleando
1590 1718 junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
1591 1719 offhei_interf, nhei_interf + offhei_interf))]]]
1592 1720
1593 1721 if noise_exist:
1594 1722 # tmp_noise = jnoise[ich] / num_prof
1595 1723 tmp_noise = jnoise[ich]
1596 1724 junkspc_interf = junkspc_interf - tmp_noise
1597 1725 #junkspc_interf[:,comp_mask_prof] = 0
1598 1726
1599 1727 jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
1600 1728 jspc_interf = jspc_interf.transpose()
1601 1729 # Calculando el espectro de interferencia promedio
1602 1730 noiseid = numpy.where(
1603 1731 jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
1604 1732 noiseid = noiseid[0]
1605 1733 cnoiseid = noiseid.size
1606 1734 interfid = numpy.where(
1607 1735 jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
1608 1736 interfid = interfid[0]
1609 1737 cinterfid = interfid.size
1610 1738
1611 1739 if (cnoiseid > 0):
1612 1740 jspc_interf[noiseid] = 0
1613 1741
1614 1742 # Expandiendo los perfiles a limpiar
1615 1743 if (cinterfid > 0):
1616 1744 new_interfid = (
1617 1745 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
1618 1746 new_interfid = numpy.asarray(new_interfid)
1619 1747 new_interfid = {x for x in new_interfid}
1620 1748 new_interfid = numpy.array(list(new_interfid))
1621 1749 new_cinterfid = new_interfid.size
1622 1750 else:
1623 1751 new_cinterfid = 0
1624 1752
1625 1753 for ip in range(new_cinterfid):
1626 1754 ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
1627 1755 jspc_interf[new_interfid[ip]
1628 1756 ] = junkspc_interf[ind[nhei_interf // 2], new_interfid[ip]]
1629 1757
1630 1758 jspectra[ich, :, ind_hei] = jspectra[ich, :,
1631 1759 ind_hei] - jspc_interf # Corregir indices
1632 1760
1633 1761 # Removiendo la interferencia del punto de mayor interferencia
1634 1762 ListAux = jspc_interf[mask_prof].tolist()
1635 1763 maxid = ListAux.index(max(ListAux))
1636 1764
1637 1765 if cinterfid > 0:
1638 1766 for ip in range(cinterfid * (interf == 2) - 1):
1639 1767 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
1640 1768 (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
1641 1769 cind = len(ind)
1642 1770
1643 1771 if (cind > 0):
1644 1772 jspectra[ich, interfid[ip], ind] = tmp_noise * \
1645 1773 (1 + (numpy.random.uniform(cind) - 0.5) /
1646 1774 numpy.sqrt(num_incoh))
1647 1775
1648 1776 ind = numpy.array([-2, -1, 1, 2])
1649 1777 xx = numpy.zeros([4, 4])
1650 1778
1651 1779 for id1 in range(4):
1652 1780 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1653 1781
1654 1782 xx_inv = numpy.linalg.inv(xx)
1655 1783 xx = xx_inv[:, 0]
1656 1784 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1657 1785 yy = jspectra[ich, mask_prof[ind], :]
1658 1786 jspectra[ich, mask_prof[maxid], :] = numpy.dot(
1659 1787 yy.transpose(), xx)
1660 1788
1661 1789 indAux = (jspectra[ich, :, :] < tmp_noise *
1662 1790 (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
1663 1791 jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
1664 1792 (1 - 1 / numpy.sqrt(num_incoh))
1665 1793
1666 1794 # Remocion de Interferencia en el Cross Spectra
1667 1795 if jcspectra is None:
1668 1796 return jspectra, jcspectra
1669 1797 num_pairs = int(jcspectra.size / (num_prof * num_hei))
1670 1798 jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
1671 1799
1672 1800 for ip in range(num_pairs):
1673 1801
1674 1802 #-------------------------------------------
1675 1803
1676 1804 cspower = numpy.abs(jcspectra[ip, mask_prof, :])
1677 1805 cspower = cspower[:, hei_interf]
1678 1806 cspower = cspower.sum(axis=0)
1679 1807
1680 1808 cspsort = cspower.ravel().argsort()
1681 1809 junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
1682 1810 offhei_interf, nhei_interf + offhei_interf))]]]
1683 1811 junkcspc_interf = junkcspc_interf.transpose()
1684 1812 jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
1685 1813
1686 1814 ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
1687 1815
1688 1816 median_real = int(numpy.median(numpy.real(
1689 1817 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1690 1818 median_imag = int(numpy.median(numpy.imag(
1691 1819 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1692 1820 comp_mask_prof = [int(e) for e in comp_mask_prof]
1693 1821 junkcspc_interf[comp_mask_prof, :] = numpy.complex(
1694 1822 median_real, median_imag)
1695 1823
1696 1824 for iprof in range(num_prof):
1697 1825 ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
1698 1826 jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf // 2]]
1699 1827
1700 1828 # Removiendo la Interferencia
1701 1829 jcspectra[ip, :, ind_hei] = jcspectra[ip,
1702 1830 :, ind_hei] - jcspc_interf
1703 1831
1704 1832 ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
1705 1833 maxid = ListAux.index(max(ListAux))
1706 1834
1707 1835 ind = numpy.array([-2, -1, 1, 2])
1708 1836 xx = numpy.zeros([4, 4])
1709 1837
1710 1838 for id1 in range(4):
1711 1839 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1712 1840
1713 1841 xx_inv = numpy.linalg.inv(xx)
1714 1842 xx = xx_inv[:, 0]
1715 1843
1716 1844 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1717 1845 yy = jcspectra[ip, mask_prof[ind], :]
1718 1846 jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
1719 1847
1720 1848 # Guardar Resultados
1721 1849 self.dataOut.data_spc = jspectra
1722 1850 self.dataOut.data_cspc = jcspectra
1723 1851
1724 1852 return 1
1725 1853
1726 1854 def run(self, dataOut, interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None, mode=1):
1727 1855
1728 1856 self.dataOut = dataOut
1729 1857
1730 1858 if mode == 1:
1731 1859 self.removeInterference(interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None)
1732 1860 elif mode == 2:
1733 1861 self.removeInterference2()
1734 1862
1735 1863 return self.dataOut
1736 1864
1737 1865
1738 1866 class IncohInt(Operation):
1739 1867
1740 1868 __profIndex = 0
1741 1869 __withOverapping = False
1742 1870
1743 1871 __byTime = False
1744 1872 __initime = None
1745 1873 __lastdatatime = None
1746 1874 __integrationtime = None
1747 1875
1748 1876 __buffer_spc = None
1749 1877 __buffer_cspc = None
1750 1878 __buffer_dc = None
1751 1879
1752 1880 __dataReady = False
1753 1881
1754 1882 __timeInterval = None
1755 1883 incohInt = 0
1756 1884 nOutliers = 0
1757 1885 n = None
1758 1886
1759 1887 def __init__(self):
1760 1888
1761 1889 Operation.__init__(self)
1762 1890
1763 1891 def setup(self, n=None, timeInterval=None, overlapping=False):
1764 1892 """
1765 1893 Set the parameters of the integration class.
1766 1894
1767 1895 Inputs:
1768 1896
1769 1897 n : Number of coherent integrations
1770 1898 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1771 1899 overlapping :
1772 1900
1773 1901 """
1774 1902
1775 1903 self.__initime = None
1776 1904 self.__lastdatatime = 0
1777 1905
1778 1906 self.__buffer_spc = 0
1779 1907 self.__buffer_cspc = 0
1780 1908 self.__buffer_dc = 0
1781 1909
1782 1910 self.__profIndex = 0
1783 1911 self.__dataReady = False
1784 1912 self.__byTime = False
1785 1913 self.incohInt = 0
1786 1914 self.nOutliers = 0
1787 1915 if n is None and timeInterval is None:
1788 1916 raise ValueError("n or timeInterval should be specified ...")
1789 1917
1790 1918 if n is not None:
1791 1919 self.n = int(n)
1792 1920 else:
1793 1921
1794 1922 self.__integrationtime = int(timeInterval)
1795 1923 self.n = None
1796 1924 self.__byTime = True
1797 1925
1798 1926 def putData(self, data_spc, data_cspc, data_dc):
1799 1927 """
1800 1928 Add a profile to the __buffer_spc and increase in one the __profileIndex
1801 1929
1802 1930 """
1803 1931 if data_spc.all() == numpy.nan :
1804 1932 print("nan ")
1805 1933 return
1806 1934 self.__buffer_spc += data_spc
1807 1935
1808 1936 if data_cspc is None:
1809 1937 self.__buffer_cspc = None
1810 1938 else:
1811 1939 self.__buffer_cspc += data_cspc
1812 1940
1813 1941 if data_dc is None:
1814 1942 self.__buffer_dc = None
1815 1943 else:
1816 1944 self.__buffer_dc += data_dc
1817 1945
1818 1946 self.__profIndex += 1
1819 1947
1820 1948 return
1821 1949
1822 1950 def pushData(self):
1823 1951 """
1824 1952 Return the sum of the last profiles and the profiles used in the sum.
1825 1953
1826 1954 Affected:
1827 1955
1828 1956 self.__profileIndex
1829 1957
1830 1958 """
1831 1959
1832 1960 data_spc = self.__buffer_spc
1833 1961 data_cspc = self.__buffer_cspc
1834 1962 data_dc = self.__buffer_dc
1835 1963 n = self.__profIndex
1836 1964
1837 1965 self.__buffer_spc = 0
1838 1966 self.__buffer_cspc = 0
1839 1967 self.__buffer_dc = 0
1840 1968
1841 1969
1842 1970 return data_spc, data_cspc, data_dc, n
1843 1971
1844 1972 def byProfiles(self, *args):
1845 1973
1846 1974 self.__dataReady = False
1847 1975 avgdata_spc = None
1848 1976 avgdata_cspc = None
1849 1977 avgdata_dc = None
1850 1978
1851 1979 self.putData(*args)
1852 1980
1853 1981 if self.__profIndex == self.n:
1854 1982
1855 1983 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1856 1984 self.n = n
1857 1985 self.__dataReady = True
1858 1986
1859 1987 return avgdata_spc, avgdata_cspc, avgdata_dc
1860 1988
1861 1989 def byTime(self, datatime, *args):
1862 1990
1863 1991 self.__dataReady = False
1864 1992 avgdata_spc = None
1865 1993 avgdata_cspc = None
1866 1994 avgdata_dc = None
1867 1995
1868 1996 self.putData(*args)
1869 1997
1870 1998 if (datatime - self.__initime) >= self.__integrationtime:
1871 1999 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1872 2000 self.n = n
1873 2001 self.__dataReady = True
1874 2002
1875 2003 return avgdata_spc, avgdata_cspc, avgdata_dc
1876 2004
1877 2005 def integrate(self, datatime, *args):
1878 2006
1879 2007 if self.__profIndex == 0:
1880 2008 self.__initime = datatime
1881 2009
1882 2010 if self.__byTime:
1883 2011 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
1884 2012 datatime, *args)
1885 2013 else:
1886 2014 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1887 2015
1888 2016 if not self.__dataReady:
1889 2017 return None, None, None, None
1890 2018
1891 2019 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
1892 2020
1893 2021 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
1894 2022 if n == 1:
1895 2023 return dataOut
1896 2024
1897 2025 if dataOut.flagNoData == True:
1898 2026 return dataOut
1899
2027
1900 2028 dataOut.flagNoData = True
1901 2029
1902 2030 if not self.isConfig:
1903 2031 self.setup(n, timeInterval, overlapping)
1904 2032 self.isConfig = True
1905 2033
1906 2034 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
1907 2035 dataOut.data_spc,
1908 2036 dataOut.data_cspc,
1909 2037 dataOut.data_dc)
1910 2038 self.incohInt += dataOut.nIncohInt
1911 2039 self.nOutliers += dataOut.data_outlier
1912 2040 if self.__dataReady:
1913
2041 #print("prof: ",dataOut.max_nIncohInt,self.__profIndex)
1914 2042 dataOut.data_spc = avgdata_spc
1915 2043 dataOut.data_cspc = avgdata_cspc
1916 2044 dataOut.data_dc = avgdata_dc
1917 2045 dataOut.nIncohInt = self.incohInt
1918 2046 dataOut.data_outlier = self.nOutliers
1919 2047 dataOut.utctime = avgdatatime
1920 2048 dataOut.flagNoData = False
1921 dataOut.max_nIncohInt = self.__profIndex
2049 dataOut.max_nIncohInt += self.__profIndex
1922 2050 self.incohInt = 0
1923 2051 self.nOutliers = 0
1924 2052 self.__profIndex = 0
1925 2053
1926 2054 return dataOut
1927 2055
1928 2056 class dopplerFlip(Operation):
1929 2057
1930 2058 def run(self, dataOut):
1931 2059 # arreglo 1: (num_chan, num_profiles, num_heights)
1932 2060 self.dataOut = dataOut
1933 2061 # JULIA-oblicua, indice 2
1934 2062 # arreglo 2: (num_profiles, num_heights)
1935 2063 jspectra = self.dataOut.data_spc[2]
1936 2064 jspectra_tmp = numpy.zeros(jspectra.shape)
1937 2065 num_profiles = jspectra.shape[0]
1938 2066 freq_dc = int(num_profiles / 2)
1939 2067 # Flip con for
1940 2068 for j in range(num_profiles):
1941 2069 jspectra_tmp[num_profiles-j-1]= jspectra[j]
1942 2070 # Intercambio perfil de DC con perfil inmediato anterior
1943 2071 jspectra_tmp[freq_dc-1]= jspectra[freq_dc-1]
1944 2072 jspectra_tmp[freq_dc]= jspectra[freq_dc]
1945 2073 # canal modificado es re-escrito en el arreglo de canales
1946 2074 self.dataOut.data_spc[2] = jspectra_tmp
1947 2075
1948 2076 return self.dataOut
Show file before | Show file after | Hide comments
@@ -1,2361 +1,2369
1 1 import sys
2 2 import numpy,math
3 3 from scipy import interpolate
4 4 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
5 5 from schainpy.model.data.jrodata import Voltage,hildebrand_sekhon
6 6 from schainpy.utils import log
7 7 from schainpy.model.io.utils import getHei_index
8 8 from time import time
9 import datetime
9 #import datetime
10 10 import numpy
11 11 #import copy
12 12 from schainpy.model.data import _noise
13 13
14 14 class VoltageProc(ProcessingUnit):
15 15
16 16 def __init__(self):
17 17
18 18 ProcessingUnit.__init__(self)
19 19
20 20 self.dataOut = Voltage()
21 21 self.flip = 1
22 22 self.setupReq = False
23 23
24 24 def run(self):
25 25 #print("running volt proc")
26 26 if self.dataIn.type == 'AMISR':
27 27 self.__updateObjFromAmisrInput()
28 28
29 29 if self.dataOut.buffer_empty:
30 30 if self.dataIn.type == 'Voltage':
31 31 self.dataOut.copy(self.dataIn)
32 32 #print("new volts reading")
33 33
34 34
35 35 def __updateObjFromAmisrInput(self):
36 36
37 37 self.dataOut.timeZone = self.dataIn.timeZone
38 38 self.dataOut.dstFlag = self.dataIn.dstFlag
39 39 self.dataOut.errorCount = self.dataIn.errorCount
40 40 self.dataOut.useLocalTime = self.dataIn.useLocalTime
41 41
42 42 self.dataOut.flagNoData = self.dataIn.flagNoData
43 43 self.dataOut.data = self.dataIn.data
44 44 self.dataOut.utctime = self.dataIn.utctime
45 45 self.dataOut.channelList = self.dataIn.channelList
46 46 #self.dataOut.timeInterval = self.dataIn.timeInterval
47 47 self.dataOut.heightList = self.dataIn.heightList
48 48 self.dataOut.nProfiles = self.dataIn.nProfiles
49 49
50 50 self.dataOut.nCohInt = self.dataIn.nCohInt
51 51 self.dataOut.ippSeconds = self.dataIn.ippSeconds
52 52 self.dataOut.frequency = self.dataIn.frequency
53 53
54 54 self.dataOut.azimuth = self.dataIn.azimuth
55 55 self.dataOut.zenith = self.dataIn.zenith
56 56
57 57 self.dataOut.beam.codeList = self.dataIn.beam.codeList
58 58 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
59 59 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
60 60
61 61
62 62 class selectChannels(Operation):
63 63
64 64 def run(self, dataOut, channelList=None):
65 65 self.channelList = channelList
66 66 if self.channelList == None:
67 67 print("Missing channelList")
68 68 return dataOut
69 69 channelIndexList = []
70 70
71 71 if type(dataOut.channelList) is not list: #leer array desde HDF5
72 72 try:
73 73 dataOut.channelList = dataOut.channelList.tolist()
74 74 except Exception as e:
75 75 print("Select Channels: ",e)
76 76 for channel in self.channelList:
77 77 if channel not in dataOut.channelList:
78 78 raise ValueError("Channel %d is not in %s" %(channel, str(dataOut.channelList)))
79 79
80 80 index = dataOut.channelList.index(channel)
81 81 channelIndexList.append(index)
82 82 dataOut = self.selectChannelsByIndex(dataOut,channelIndexList)
83 83 return dataOut
84 84
85 85 def selectChannelsByIndex(self, dataOut, channelIndexList):
86 86 """
87 87 Selecciona un bloque de datos en base a canales segun el channelIndexList
88 88
89 89 Input:
90 90 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
91 91
92 92 Affected:
93 93 dataOut.data
94 94 dataOut.channelIndexList
95 95 dataOut.nChannels
96 96 dataOut.m_ProcessingHeader.totalSpectra
97 97 dataOut.systemHeaderObj.numChannels
98 98 dataOut.m_ProcessingHeader.blockSize
99 99
100 100 Return:
101 101 None
102 102 """
103 103 #print("selectChannelsByIndex")
104 104 # for channelIndex in channelIndexList:
105 105 # if channelIndex not in dataOut.channelIndexList:
106 106 # raise ValueError("The value %d in channelIndexList is not valid" %channelIndex)
107 107
108 108 if dataOut.type == 'Voltage':
109 109 if dataOut.flagDataAsBlock:
110 110 """
111 111 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
112 112 """
113 113 data = dataOut.data[channelIndexList,:,:]
114 114 else:
115 115 data = dataOut.data[channelIndexList,:]
116 116
117 117 dataOut.data = data
118 118 # dataOut.channelList = [dataOut.channelList[i] for i in channelIndexList]
119 119 dataOut.channelList = range(len(channelIndexList))
120 120
121 121 elif dataOut.type == 'Spectra':
122 122 if hasattr(dataOut, 'data_spc'):
123 123 if dataOut.data_spc is None:
124 124 raise ValueError("data_spc is None")
125 125 return dataOut
126 126 else:
127 127 data_spc = dataOut.data_spc[channelIndexList, :]
128 128 dataOut.data_spc = data_spc
129 129
130 130 # if hasattr(dataOut, 'data_dc') :# and
131 131 # if dataOut.data_dc is None:
132 132 # raise ValueError("data_dc is None")
133 133 # return dataOut
134 134 # else:
135 135 # data_dc = dataOut.data_dc[channelIndexList, :]
136 136 # dataOut.data_dc = data_dc
137 137 # dataOut.channelList = [dataOut.channelList[i] for i in channelIndexList]
138 138 dataOut.channelList = channelIndexList
139 139 dataOut = self.__selectPairsByChannel(dataOut,channelIndexList)
140 140
141 141 return dataOut
142 142
143 143 def __selectPairsByChannel(self, dataOut, channelList=None):
144 144 #print("__selectPairsByChannel")
145 145 if channelList == None:
146 146 return
147 147
148 148 pairsIndexListSelected = []
149 149 for pairIndex in dataOut.pairsIndexList:
150 150 # First pair
151 151 if dataOut.pairsList[pairIndex][0] not in channelList:
152 152 continue
153 153 # Second pair
154 154 if dataOut.pairsList[pairIndex][1] not in channelList:
155 155 continue
156 156
157 157 pairsIndexListSelected.append(pairIndex)
158 158 if not pairsIndexListSelected:
159 159 dataOut.data_cspc = None
160 160 dataOut.pairsList = []
161 161 return
162 162
163 163 dataOut.data_cspc = dataOut.data_cspc[pairsIndexListSelected]
164 164 dataOut.pairsList = [dataOut.pairsList[i]
165 165 for i in pairsIndexListSelected]
166 166
167 167 return dataOut
168 168
169 169 class selectHeights(Operation):
170 170
171 171 def run(self, dataOut, minHei=None, maxHei=None, minIndex=None, maxIndex=None):
172 172 """
173 173 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
174 174 minHei <= height <= maxHei
175 175
176 176 Input:
177 177 minHei : valor minimo de altura a considerar
178 178 maxHei : valor maximo de altura a considerar
179 179
180 180 Affected:
181 181 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
182 182
183 183 Return:
184 184 1 si el metodo se ejecuto con exito caso contrario devuelve 0
185 185 """
186 186
187 187 self.dataOut = dataOut
188 188
189 189 if minHei and maxHei:
190 190
191 191 if (minHei < dataOut.heightList[0]):
192 192 minHei = dataOut.heightList[0]
193 193
194 194 if (maxHei > dataOut.heightList[-1]):
195 195 maxHei = dataOut.heightList[-1]
196 196
197 197 minIndex = 0
198 198 maxIndex = 0
199 199 heights = dataOut.heightList
200 200
201 201 inda = numpy.where(heights >= minHei)
202 202 indb = numpy.where(heights <= maxHei)
203 203
204 204 try:
205 205 minIndex = inda[0][0]
206 206 except:
207 207 minIndex = 0
208 208
209 209 try:
210 210 maxIndex = indb[0][-1]
211 211 except:
212 212 maxIndex = len(heights)
213 213
214 214 self.selectHeightsByIndex(minIndex, maxIndex)
215 215
216 216 return dataOut
217 217
218 218 def selectHeightsByIndex(self, minIndex, maxIndex):
219 219 """
220 220 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
221 221 minIndex <= index <= maxIndex
222 222
223 223 Input:
224 224 minIndex : valor de indice minimo de altura a considerar
225 225 maxIndex : valor de indice maximo de altura a considerar
226 226
227 227 Affected:
228 228 self.dataOut.data
229 229 self.dataOut.heightList
230 230
231 231 Return:
232 232 1 si el metodo se ejecuto con exito caso contrario devuelve 0
233 233 """
234 234
235 235 if self.dataOut.type == 'Voltage':
236 236 if (minIndex < 0) or (minIndex > maxIndex):
237 237 raise ValueError("Height index range (%d,%d) is not valid" % (minIndex, maxIndex))
238 238
239 239 if (maxIndex >= self.dataOut.nHeights):
240 240 maxIndex = self.dataOut.nHeights
241 241
242 242 #voltage
243 243 if self.dataOut.flagDataAsBlock:
244 244 """
245 245 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
246 246 """
247 247 data = self.dataOut.data[:,:, minIndex:maxIndex]
248 248 else:
249 249 data = self.dataOut.data[:, minIndex:maxIndex]
250 250
251 251 # firstHeight = self.dataOut.heightList[minIndex]
252 252
253 253 self.dataOut.data = data
254 254 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
255 255
256 256 if self.dataOut.nHeights <= 1:
257 257 raise ValueError("selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights))
258 258 elif self.dataOut.type == 'Spectra':
259 259 if (minIndex < 0) or (minIndex > maxIndex):
260 260 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (
261 261 minIndex, maxIndex))
262 262
263 263 if (maxIndex >= self.dataOut.nHeights):
264 264 maxIndex = self.dataOut.nHeights - 1
265 265
266 266 # Spectra
267 267 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
268 268
269 269 data_cspc = None
270 270 if self.dataOut.data_cspc is not None:
271 271 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
272 272
273 273 data_dc = None
274 274 if self.dataOut.data_dc is not None:
275 275 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
276 276
277 277 self.dataOut.data_spc = data_spc
278 278 self.dataOut.data_cspc = data_cspc
279 279 self.dataOut.data_dc = data_dc
280 280
281 281 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
282 282
283 283 return 1
284 284
285 285
286 286 class filterByHeights(Operation):
287 287
288 288 def run(self, dataOut, window):
289 289
290 290 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
291 291
292 292 if window == None:
293 293 window = (dataOut.radarControllerHeaderObj.txA/dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
294 294
295 295 newdelta = deltaHeight * window
296 296 r = dataOut.nHeights % window
297 297 newheights = (dataOut.nHeights-r)/window
298 298
299 299 if newheights <= 1:
300 300 raise ValueError("filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(dataOut.nHeights, window))
301 301
302 302 if dataOut.flagDataAsBlock:
303 303 """
304 304 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
305 305 """
306 306 buffer = dataOut.data[:, :, 0:int(dataOut.nHeights-r)]
307 307 buffer = buffer.reshape(dataOut.nChannels, dataOut.nProfiles, int(dataOut.nHeights/window), window)
308 308 buffer = numpy.sum(buffer,3)
309 309
310 310 else:
311 311 buffer = dataOut.data[:,0:int(dataOut.nHeights-r)]
312 312 buffer = buffer.reshape(dataOut.nChannels,int(dataOut.nHeights/window),int(window))
313 313 buffer = numpy.sum(buffer,2)
314 314
315 315 dataOut.data = buffer
316 316 dataOut.heightList = dataOut.heightList[0] + numpy.arange( newheights )*newdelta
317 317 dataOut.windowOfFilter = window
318 318
319 319 return dataOut
320 320
321 321
322 322 class setH0(Operation):
323 323
324 324 def run(self, dataOut, h0, deltaHeight = None):
325 325
326 326 if not deltaHeight:
327 327 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
328 328
329 329 nHeights = dataOut.nHeights
330 330
331 331 newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
332 332
333 333 dataOut.heightList = newHeiRange
334 334
335 335 return dataOut
336 336
337 337
338 338 class deFlip(Operation):
339 339
340 340 def run(self, dataOut, channelList = []):
341 341
342 342 data = dataOut.data.copy()
343 343
344 344 if dataOut.flagDataAsBlock:
345 345 flip = self.flip
346 346 profileList = list(range(dataOut.nProfiles))
347 347
348 348 if not channelList:
349 349 for thisProfile in profileList:
350 350 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
351 351 flip *= -1.0
352 352 else:
353 353 for thisChannel in channelList:
354 354 if thisChannel not in dataOut.channelList:
355 355 continue
356 356
357 357 for thisProfile in profileList:
358 358 data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
359 359 flip *= -1.0
360 360
361 361 self.flip = flip
362 362
363 363 else:
364 364 if not channelList:
365 365 data[:,:] = data[:,:]*self.flip
366 366 else:
367 367 for thisChannel in channelList:
368 368 if thisChannel not in dataOut.channelList:
369 369 continue
370 370
371 371 data[thisChannel,:] = data[thisChannel,:]*self.flip
372 372
373 373 self.flip *= -1.
374 374
375 375 dataOut.data = data
376 376
377 377 return dataOut
378 378
379 379
380 380 class setAttribute(Operation):
381 381 '''
382 382 Set an arbitrary attribute(s) to dataOut
383 383 '''
384 384
385 385 def __init__(self):
386 386
387 387 Operation.__init__(self)
388 388 self._ready = False
389 389
390 390 def run(self, dataOut, **kwargs):
391 391
392 392 for key, value in kwargs.items():
393 393 setattr(dataOut, key, value)
394 394
395 395 return dataOut
396 396
397 397
398 398 @MPDecorator
399 399 class printAttribute(Operation):
400 400 '''
401 401 Print an arbitrary attribute of dataOut
402 402 '''
403 403
404 404 def __init__(self):
405 405
406 406 Operation.__init__(self)
407 407
408 408 def run(self, dataOut, attributes):
409 409
410 410 if isinstance(attributes, str):
411 411 attributes = [attributes]
412 412 for attr in attributes:
413 413 if hasattr(dataOut, attr):
414 414 log.log(getattr(dataOut, attr), attr)
415 415
416 416
417 417 class interpolateHeights(Operation):
418 418
419 419 def run(self, dataOut, topLim, botLim):
420 420 #69 al 72 para julia
421 421 #82-84 para meteoros
422 422 if len(numpy.shape(dataOut.data))==2:
423 423 sampInterp = (dataOut.data[:,botLim-1] + dataOut.data[:,topLim+1])/2
424 424 sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
425 425 #dataOut.data[:,botLim:limSup+1] = sampInterp
426 426 dataOut.data[:,botLim:topLim+1] = sampInterp
427 427 else:
428 428 nHeights = dataOut.data.shape[2]
429 429 x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
430 430 y = dataOut.data[:,:,list(range(botLim))+list(range(topLim+1,nHeights))]
431 431 f = interpolate.interp1d(x, y, axis = 2)
432 432 xnew = numpy.arange(botLim,topLim+1)
433 433 ynew = f(xnew)
434 434 dataOut.data[:,:,botLim:topLim+1] = ynew
435 435
436 436 return dataOut
437 437
438 438
439 439 class CohInt(Operation):
440 440
441 441 isConfig = False
442 442 __profIndex = 0
443 443 __byTime = False
444 444 __initime = None
445 445 __lastdatatime = None
446 446 __integrationtime = None
447 447 __buffer = None
448 448 __bufferStride = []
449 449 __dataReady = False
450 450 __profIndexStride = 0
451 451 __dataToPutStride = False
452 452 n = None
453 453
454 454 def __init__(self, **kwargs):
455 455
456 456 Operation.__init__(self, **kwargs)
457 457
458 458 def setup(self, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False):
459 459 """
460 460 Set the parameters of the integration class.
461 461
462 462 Inputs:
463 463
464 464 n : Number of coherent integrations
465 465 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
466 466 overlapping :
467 467 """
468 468
469 469 self.__initime = None
470 470 self.__lastdatatime = 0
471 471 self.__buffer = None
472 472 self.__dataReady = False
473 473 self.byblock = byblock
474 474 self.stride = stride
475 475
476 476 if n == None and timeInterval == None:
477 477 raise ValueError("n or timeInterval should be specified ...")
478 478
479 479 if n != None:
480 480 self.n = n
481 481 self.__byTime = False
482 482 else:
483 483 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
484 484 self.n = 9999
485 485 self.__byTime = True
486 486
487 487 if overlapping:
488 488 self.__withOverlapping = True
489 489 self.__buffer = None
490 490 else:
491 491 self.__withOverlapping = False
492 492 self.__buffer = 0
493 493
494 494 self.__profIndex = 0
495 495
496 496 def putData(self, data):
497 497
498 498 """
499 499 Add a profile to the __buffer and increase in one the __profileIndex
500 500
501 501 """
502 502
503 503 if not self.__withOverlapping:
504 504 self.__buffer += data.copy()
505 505 self.__profIndex += 1
506 506 return
507 507
508 508 #Overlapping data
509 509 nChannels, nHeis = data.shape
510 510 data = numpy.reshape(data, (1, nChannels, nHeis))
511 511
512 512 #If the buffer is empty then it takes the data value
513 513 if self.__buffer is None:
514 514 self.__buffer = data
515 515 self.__profIndex += 1
516 516 return
517 517
518 518 #If the buffer length is lower than n then stakcing the data value
519 519 if self.__profIndex < self.n:
520 520 self.__buffer = numpy.vstack((self.__buffer, data))
521 521 self.__profIndex += 1
522 522 return
523 523
524 524 #If the buffer length is equal to n then replacing the last buffer value with the data value
525 525 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
526 526 self.__buffer[self.n-1] = data
527 527 self.__profIndex = self.n
528 528 return
529 529
530 530
531 531 def pushData(self):
532 532 """
533 533 Return the sum of the last profiles and the profiles used in the sum.
534 534
535 535 Affected:
536 536
537 537 self.__profileIndex
538 538
539 539 """
540 540
541 541 if not self.__withOverlapping:
542 542 data = self.__buffer
543 543 n = self.__profIndex
544 544
545 545 self.__buffer = 0
546 546 self.__profIndex = 0
547 547
548 548 return data, n
549 549
550 550 #Integration with Overlapping
551 551 data = numpy.sum(self.__buffer, axis=0)
552 552 # print data
553 553 # raise
554 554 n = self.__profIndex
555 555
556 556 return data, n
557 557
558 558 def byProfiles(self, data):
559 559
560 560 self.__dataReady = False
561 561 avgdata = None
562 562 # n = None
563 563 # print data
564 564 # raise
565 565 self.putData(data)
566 566
567 567 if self.__profIndex == self.n:
568 568 avgdata, n = self.pushData()
569 569 self.__dataReady = True
570 570
571 571 return avgdata
572 572
573 573 def byTime(self, data, datatime):
574 574
575 575 self.__dataReady = False
576 576 avgdata = None
577 577 n = None
578 578
579 579 self.putData(data)
580 580
581 581 if (datatime - self.__initime) >= self.__integrationtime:
582 582 avgdata, n = self.pushData()
583 583 self.n = n
584 584 self.__dataReady = True
585 585
586 586 return avgdata
587 587
588 588 def integrateByStride(self, data, datatime):
589 589 # print data
590 590 if self.__profIndex == 0:
591 591 self.__buffer = [[data.copy(), datatime]]
592 592 else:
593 593 self.__buffer.append([data.copy(),datatime])
594 594 self.__profIndex += 1
595 595 self.__dataReady = False
596 596
597 597 if self.__profIndex == self.n * self.stride :
598 598 self.__dataToPutStride = True
599 599 self.__profIndexStride = 0
600 600 self.__profIndex = 0
601 601 self.__bufferStride = []
602 602 for i in range(self.stride):
603 603 current = self.__buffer[i::self.stride]
604 604 data = numpy.sum([t[0] for t in current], axis=0)
605 605 avgdatatime = numpy.average([t[1] for t in current])
606 606 # print data
607 607 self.__bufferStride.append((data, avgdatatime))
608 608
609 609 if self.__dataToPutStride:
610 610 self.__dataReady = True
611 611 self.__profIndexStride += 1
612 612 if self.__profIndexStride == self.stride:
613 613 self.__dataToPutStride = False
614 614 # print self.__bufferStride[self.__profIndexStride - 1]
615 615 # raise
616 616 return self.__bufferStride[self.__profIndexStride - 1]
617 617
618 618
619 619 return None, None
620 620
621 621 def integrate(self, data, datatime=None):
622 622
623 623 if self.__initime == None:
624 624 self.__initime = datatime
625 625
626 626 if self.__byTime:
627 627 avgdata = self.byTime(data, datatime)
628 628 else:
629 629 avgdata = self.byProfiles(data)
630 630
631 631
632 632 self.__lastdatatime = datatime
633 633
634 634 if avgdata is None:
635 635 return None, None
636 636
637 637 avgdatatime = self.__initime
638 638
639 639 deltatime = datatime - self.__lastdatatime
640 640
641 641 if not self.__withOverlapping:
642 642 self.__initime = datatime
643 643 else:
644 644 self.__initime += deltatime
645 645
646 646 return avgdata, avgdatatime
647 647
648 648 def integrateByBlock(self, dataOut):
649 649
650 650 times = int(dataOut.data.shape[1]/self.n)
651 651 avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
652 652
653 653 id_min = 0
654 654 id_max = self.n
655 655
656 656 for i in range(times):
657 657 junk = dataOut.data[:,id_min:id_max,:]
658 658 avgdata[:,i,:] = junk.sum(axis=1)
659 659 id_min += self.n
660 660 id_max += self.n
661 661
662 662 timeInterval = dataOut.ippSeconds*self.n
663 663 avgdatatime = (times - 1) * timeInterval + dataOut.utctime
664 664 self.__dataReady = True
665 665 return avgdata, avgdatatime
666 666
667 667 def run(self, dataOut, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False, **kwargs):
668 668
669 669 if not self.isConfig:
670 670 self.setup(n=n, stride=stride, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
671 671 self.isConfig = True
672 672
673 673 if dataOut.flagDataAsBlock:
674 674 """
675 675 Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
676 676 """
677 677 avgdata, avgdatatime = self.integrateByBlock(dataOut)
678 678 dataOut.nProfiles /= self.n
679 679 else:
680 680 if stride is None:
681 681 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
682 682 else:
683 683 avgdata, avgdatatime = self.integrateByStride(dataOut.data, dataOut.utctime)
684 684
685 685
686 686 # dataOut.timeInterval *= n
687 687 dataOut.flagNoData = True
688 688
689 689 if self.__dataReady:
690 690 dataOut.data = avgdata
691 691 if not dataOut.flagCohInt:
692 692 dataOut.nCohInt *= self.n
693 693 dataOut.flagCohInt = True
694 694 dataOut.utctime = avgdatatime
695 695 # print avgdata, avgdatatime
696 696 # raise
697 697 # dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
698 698 dataOut.flagNoData = False
699 699 return dataOut
700 700
701 701 class Decoder(Operation):
702 702
703 703 isConfig = False
704 704 __profIndex = 0
705 705
706 706 code = None
707 707
708 708 nCode = None
709 709 nBaud = None
710 710
711 711 def __init__(self, **kwargs):
712 712
713 713 Operation.__init__(self, **kwargs)
714 714
715 715 self.times = None
716 716 self.osamp = None
717 717 # self.__setValues = False
718 718 self.isConfig = False
719 719 self.setupReq = False
720 720 def setup(self, code, osamp, dataOut):
721 721
722 722 self.__profIndex = 0
723 723
724 724 self.code = code
725 725
726 726 self.nCode = len(code)
727 727 self.nBaud = len(code[0])
728 728 if (osamp != None) and (osamp >1):
729 729 self.osamp = osamp
730 730 self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
731 731 self.nBaud = self.nBaud*self.osamp
732 732
733 733 self.__nChannels = dataOut.nChannels
734 734 self.__nProfiles = dataOut.nProfiles
735 735 self.__nHeis = dataOut.nHeights
736 736
737 737 if self.__nHeis < self.nBaud:
738 738 raise ValueError('Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud))
739 739
740 740 #Frequency
741 741 __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
742 742
743 743 __codeBuffer[:,0:self.nBaud] = self.code
744 744
745 745 self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
746 746
747 747 if dataOut.flagDataAsBlock:
748 748
749 749 self.ndatadec = self.__nHeis #- self.nBaud + 1
750 750
751 751 self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
752 752
753 753 else:
754 754
755 755 #Time
756 756 self.ndatadec = self.__nHeis #- self.nBaud + 1
757 757
758 758 self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
759 759
760 760 def __convolutionInFreq(self, data):
761 761
762 762 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
763 763
764 764 fft_data = numpy.fft.fft(data, axis=1)
765 765
766 766 conv = fft_data*fft_code
767 767
768 768 data = numpy.fft.ifft(conv,axis=1)
769 769
770 770 return data
771 771
772 772 def __convolutionInFreqOpt(self, data):
773 773
774 774 raise NotImplementedError
775 775
776 776 def __convolutionInTime(self, data):
777 777
778 778 code = self.code[self.__profIndex]
779 779 for i in range(self.__nChannels):
780 780 self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
781 781
782 782 return self.datadecTime
783 783
784 784 def __convolutionByBlockInTime(self, data):
785 785
786 786 repetitions = int(self.__nProfiles / self.nCode)
787 787 junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
788 788 junk = junk.flatten()
789 789 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
790 790 profilesList = range(self.__nProfiles)
791 791
792 792 for i in range(self.__nChannels):
793 793 for j in profilesList:
794 794 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
795 795 return self.datadecTime
796 796
797 797 def __convolutionByBlockInFreq(self, data):
798 798
799 799 raise NotImplementedError("Decoder by frequency fro Blocks not implemented")
800 800
801 801
802 802 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
803 803
804 804 fft_data = numpy.fft.fft(data, axis=2)
805 805
806 806 conv = fft_data*fft_code
807 807
808 808 data = numpy.fft.ifft(conv,axis=2)
809 809
810 810 return data
811 811
812 812
813 813 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
814 814
815 815 if dataOut.flagDecodeData:
816 816 print("This data is already decoded, recoding again ...")
817 817
818 818 if not self.isConfig:
819 819
820 820 if code is None:
821 821 if dataOut.code is None:
822 822 raise ValueError("Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type)
823 823
824 824 code = dataOut.code
825 825 else:
826 826 code = numpy.array(code).reshape(nCode,nBaud)
827 827 self.setup(code, osamp, dataOut)
828 828
829 829 self.isConfig = True
830 830
831 831 if mode == 3:
832 832 sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
833 833
834 834 if times != None:
835 835 sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
836 836
837 837 if self.code is None:
838 838 print("Fail decoding: Code is not defined.")
839 839 return
840 840
841 841 self.__nProfiles = dataOut.nProfiles
842 842 datadec = None
843 843
844 844 if mode == 3:
845 845 mode = 0
846 846
847 847 if dataOut.flagDataAsBlock:
848 848 """
849 849 Decoding when data have been read as block,
850 850 """
851 851
852 852 if mode == 0:
853 853 datadec = self.__convolutionByBlockInTime(dataOut.data)
854 854 if mode == 1:
855 855 datadec = self.__convolutionByBlockInFreq(dataOut.data)
856 856 else:
857 857 """
858 858 Decoding when data have been read profile by profile
859 859 """
860 860 if mode == 0:
861 861 datadec = self.__convolutionInTime(dataOut.data)
862 862
863 863 if mode == 1:
864 864 datadec = self.__convolutionInFreq(dataOut.data)
865 865
866 866 if mode == 2:
867 867 datadec = self.__convolutionInFreqOpt(dataOut.data)
868 868
869 869 if datadec is None:
870 870 raise ValueError("Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode)
871 871
872 872 dataOut.code = self.code
873 873 dataOut.nCode = self.nCode
874 874 dataOut.nBaud = self.nBaud
875 875
876 876 dataOut.data = datadec
877 877
878 878 dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
879 879
880 880 dataOut.flagDecodeData = True #asumo q la data esta decodificada
881 881
882 882 if self.__profIndex == self.nCode-1:
883 883 self.__profIndex = 0
884 884 return dataOut
885 885
886 886 self.__profIndex += 1
887 887
888 888 return dataOut
889 889 # dataOut.flagDeflipData = True #asumo q la data no esta sin flip
890 890
891 891
892 892 class ProfileConcat(Operation):
893 893
894 894 isConfig = False
895 895 buffer = None
896 896
897 897 def __init__(self, **kwargs):
898 898
899 899 Operation.__init__(self, **kwargs)
900 900 self.profileIndex = 0
901 901
902 902 def reset(self):
903 903 self.buffer = numpy.zeros_like(self.buffer)
904 904 self.start_index = 0
905 905 self.times = 1
906 906
907 907 def setup(self, data, m, n=1):
908 908 self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
909 909 self.nHeights = data.shape[1]#.nHeights
910 910 self.start_index = 0
911 911 self.times = 1
912 912
913 913 def concat(self, data):
914 914
915 915 self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
916 916 self.start_index = self.start_index + self.nHeights
917 917
918 918 def run(self, dataOut, m):
919 919 dataOut.flagNoData = True
920 920
921 921 if not self.isConfig:
922 922 self.setup(dataOut.data, m, 1)
923 923 self.isConfig = True
924 924
925 925 if dataOut.flagDataAsBlock:
926 926 raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
927 927
928 928 else:
929 929 self.concat(dataOut.data)
930 930 self.times += 1
931 931 if self.times > m:
932 932 dataOut.data = self.buffer
933 933 self.reset()
934 934 dataOut.flagNoData = False
935 935 # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
936 936 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
937 937 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
938 938 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
939 939 dataOut.ippSeconds *= m
940 940 return dataOut
941 941
942 942 class ProfileSelector(Operation):
943 943
944 944 profileIndex = None
945 945 # Tamanho total de los perfiles
946 946 nProfiles = None
947 947
948 948 def __init__(self, **kwargs):
949 949
950 950 Operation.__init__(self, **kwargs)
951 951 self.profileIndex = 0
952 952
953 953 def incProfileIndex(self):
954 954
955 955 self.profileIndex += 1
956 956
957 957 if self.profileIndex >= self.nProfiles:
958 958 self.profileIndex = 0
959 959
960 960 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
961 961
962 962 if profileIndex < minIndex:
963 963 return False
964 964
965 965 if profileIndex > maxIndex:
966 966 return False
967 967
968 968 return True
969 969
970 970 def isThisProfileInList(self, profileIndex, profileList):
971 971
972 972 if profileIndex not in profileList:
973 973 return False
974 974
975 975 return True
976 976
977 977 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
978 978
979 979 """
980 980 ProfileSelector:
981 981
982 982 Inputs:
983 983 profileList : Index of profiles selected. Example: profileList = (0,1,2,7,8)
984 984
985 985 profileRangeList : Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
986 986
987 987 rangeList : List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
988 988
989 989 """
990 990
991 991 if rangeList is not None:
992 992 if type(rangeList[0]) not in (tuple, list):
993 993 rangeList = [rangeList]
994 994
995 995 dataOut.flagNoData = True
996 996
997 997 if dataOut.flagDataAsBlock:
998 998 """
999 999 data dimension = [nChannels, nProfiles, nHeis]
1000 1000 """
1001 1001 if profileList != None:
1002 1002 dataOut.data = dataOut.data[:,profileList,:]
1003 1003
1004 1004 if profileRangeList != None:
1005 1005 minIndex = profileRangeList[0]
1006 1006 maxIndex = profileRangeList[1]
1007 1007 profileList = list(range(minIndex, maxIndex+1))
1008 1008
1009 1009 dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
1010 1010
1011 1011 if rangeList != None:
1012 1012
1013 1013 profileList = []
1014 1014
1015 1015 for thisRange in rangeList:
1016 1016 minIndex = thisRange[0]
1017 1017 maxIndex = thisRange[1]
1018 1018
1019 1019 profileList.extend(list(range(minIndex, maxIndex+1)))
1020 1020
1021 1021 dataOut.data = dataOut.data[:,profileList,:]
1022 1022
1023 1023 dataOut.nProfiles = len(profileList)
1024 1024 dataOut.profileIndex = dataOut.nProfiles - 1
1025 1025 dataOut.flagNoData = False
1026 1026
1027 1027 return dataOut
1028 1028
1029 1029 """
1030 1030 data dimension = [nChannels, nHeis]
1031 1031 """
1032 1032
1033 1033 if profileList != None:
1034 1034
1035 1035 if self.isThisProfileInList(dataOut.profileIndex, profileList):
1036 1036
1037 1037 self.nProfiles = len(profileList)
1038 1038 dataOut.nProfiles = self.nProfiles
1039 1039 dataOut.profileIndex = self.profileIndex
1040 1040 dataOut.flagNoData = False
1041 1041
1042 1042 self.incProfileIndex()
1043 1043 return dataOut
1044 1044
1045 1045 if profileRangeList != None:
1046 1046
1047 1047 minIndex = profileRangeList[0]
1048 1048 maxIndex = profileRangeList[1]
1049 1049
1050 1050 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
1051 1051
1052 1052 self.nProfiles = maxIndex - minIndex + 1
1053 1053 dataOut.nProfiles = self.nProfiles
1054 1054 dataOut.profileIndex = self.profileIndex
1055 1055 dataOut.flagNoData = False
1056 1056
1057 1057 self.incProfileIndex()
1058 1058 return dataOut
1059 1059
1060 1060 if rangeList != None:
1061 1061
1062 1062 nProfiles = 0
1063 1063
1064 1064 for thisRange in rangeList:
1065 1065 minIndex = thisRange[0]
1066 1066 maxIndex = thisRange[1]
1067 1067
1068 1068 nProfiles += maxIndex - minIndex + 1
1069 1069
1070 1070 for thisRange in rangeList:
1071 1071
1072 1072 minIndex = thisRange[0]
1073 1073 maxIndex = thisRange[1]
1074 1074
1075 1075 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
1076 1076
1077 1077 self.nProfiles = nProfiles
1078 1078 dataOut.nProfiles = self.nProfiles
1079 1079 dataOut.profileIndex = self.profileIndex
1080 1080 dataOut.flagNoData = False
1081 1081
1082 1082 self.incProfileIndex()
1083 1083
1084 1084 break
1085 1085
1086 1086 return dataOut
1087 1087
1088 1088
1089 1089 if beam != None: #beam is only for AMISR data
1090 1090 if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
1091 1091 dataOut.flagNoData = False
1092 1092 dataOut.profileIndex = self.profileIndex
1093 1093
1094 1094 self.incProfileIndex()
1095 1095
1096 1096 return dataOut
1097 1097
1098 1098 raise ValueError("ProfileSelector needs profileList, profileRangeList or rangeList parameter")
1099 1099
1100 1100
1101 1101 class Reshaper(Operation):
1102 1102
1103 1103 def __init__(self, **kwargs):
1104 1104
1105 1105 Operation.__init__(self, **kwargs)
1106 1106
1107 1107 self.__buffer = None
1108 1108 self.__nitems = 0
1109 1109
1110 1110 def __appendProfile(self, dataOut, nTxs):
1111 1111
1112 1112 if self.__buffer is None:
1113 1113 shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
1114 1114 self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
1115 1115
1116 1116 ini = dataOut.nHeights * self.__nitems
1117 1117 end = ini + dataOut.nHeights
1118 1118
1119 1119 self.__buffer[:, ini:end] = dataOut.data
1120 1120
1121 1121 self.__nitems += 1
1122 1122
1123 1123 return int(self.__nitems*nTxs)
1124 1124
1125 1125 def __getBuffer(self):
1126 1126
1127 1127 if self.__nitems == int(1./self.__nTxs):
1128 1128
1129 1129 self.__nitems = 0
1130 1130
1131 1131 return self.__buffer.copy()
1132 1132
1133 1133 return None
1134 1134
1135 1135 def __checkInputs(self, dataOut, shape, nTxs):
1136 1136
1137 1137 if shape is None and nTxs is None:
1138 1138 raise ValueError("Reshaper: shape of factor should be defined")
1139 1139
1140 1140 if nTxs:
1141 1141 if nTxs < 0:
1142 1142 raise ValueError("nTxs should be greater than 0")
1143 1143
1144 1144 if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
1145 1145 raise ValueError("nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs)))
1146 1146
1147 1147 shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
1148 1148
1149 1149 return shape, nTxs
1150 1150
1151 1151 if len(shape) != 2 and len(shape) != 3:
1152 1152 raise ValueError("shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights))
1153 1153
1154 1154 if len(shape) == 2:
1155 1155 shape_tuple = [dataOut.nChannels]
1156 1156 shape_tuple.extend(shape)
1157 1157 else:
1158 1158 shape_tuple = list(shape)
1159 1159
1160 1160 nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
1161 1161
1162 1162 return shape_tuple, nTxs
1163 1163
1164 1164 def run(self, dataOut, shape=None, nTxs=None):
1165 1165
1166 1166 shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
1167 1167
1168 1168 dataOut.flagNoData = True
1169 1169 profileIndex = None
1170 1170
1171 1171 if dataOut.flagDataAsBlock:
1172 1172
1173 1173 dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
1174 1174 dataOut.flagNoData = False
1175 1175
1176 1176 profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
1177 1177
1178 1178 else:
1179 1179
1180 1180 if self.__nTxs < 1:
1181 1181
1182 1182 self.__appendProfile(dataOut, self.__nTxs)
1183 1183 new_data = self.__getBuffer()
1184 1184
1185 1185 if new_data is not None:
1186 1186 dataOut.data = new_data
1187 1187 dataOut.flagNoData = False
1188 1188
1189 1189 profileIndex = dataOut.profileIndex*nTxs
1190 1190
1191 1191 else:
1192 1192 raise ValueError("nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)")
1193 1193
1194 1194 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1195 1195
1196 1196 dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
1197 1197
1198 1198 dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
1199 1199
1200 1200 dataOut.profileIndex = profileIndex
1201 1201
1202 1202 dataOut.ippSeconds /= self.__nTxs
1203 1203
1204 1204 return dataOut
1205 1205
1206 1206 class SplitProfiles(Operation):
1207 1207
1208 1208 def __init__(self, **kwargs):
1209 1209
1210 1210 Operation.__init__(self, **kwargs)
1211 1211
1212 1212 def run(self, dataOut, n):
1213 1213
1214 1214 dataOut.flagNoData = True
1215 1215 profileIndex = None
1216 1216
1217 1217 if dataOut.flagDataAsBlock:
1218 1218
1219 1219 #nchannels, nprofiles, nsamples
1220 1220 shape = dataOut.data.shape
1221 1221
1222 1222 if shape[2] % n != 0:
1223 1223 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[2]))
1224 1224
1225 1225 new_shape = shape[0], shape[1]*n, int(shape[2]/n)
1226 1226
1227 1227 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1228 1228 dataOut.flagNoData = False
1229 1229
1230 1230 profileIndex = int(dataOut.nProfiles/n) - 1
1231 1231
1232 1232 else:
1233 1233
1234 1234 raise ValueError("Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)")
1235 1235
1236 1236 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1237 1237
1238 1238 dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
1239 1239
1240 1240 dataOut.nProfiles = int(dataOut.nProfiles*n)
1241 1241
1242 1242 dataOut.profileIndex = profileIndex
1243 1243
1244 1244 dataOut.ippSeconds /= n
1245 1245
1246 1246 return dataOut
1247 1247
1248 1248 class CombineProfiles(Operation):
1249 1249 def __init__(self, **kwargs):
1250 1250
1251 1251 Operation.__init__(self, **kwargs)
1252 1252
1253 1253 self.__remData = None
1254 1254 self.__profileIndex = 0
1255 1255
1256 1256 def run(self, dataOut, n):
1257 1257
1258 1258 dataOut.flagNoData = True
1259 1259 profileIndex = None
1260 1260
1261 1261 if dataOut.flagDataAsBlock:
1262 1262
1263 1263 #nchannels, nprofiles, nsamples
1264 1264 shape = dataOut.data.shape
1265 1265 new_shape = shape[0], shape[1]/n, shape[2]*n
1266 1266
1267 1267 if shape[1] % n != 0:
1268 1268 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[1]))
1269 1269
1270 1270 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1271 1271 dataOut.flagNoData = False
1272 1272
1273 1273 profileIndex = int(dataOut.nProfiles*n) - 1
1274 1274
1275 1275 else:
1276 1276
1277 1277 #nchannels, nsamples
1278 1278 if self.__remData is None:
1279 1279 newData = dataOut.data
1280 1280 else:
1281 1281 newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
1282 1282
1283 1283 self.__profileIndex += 1
1284 1284
1285 1285 if self.__profileIndex < n:
1286 1286 self.__remData = newData
1287 1287 #continue
1288 1288 return
1289 1289
1290 1290 self.__profileIndex = 0
1291 1291 self.__remData = None
1292 1292
1293 1293 dataOut.data = newData
1294 1294 dataOut.flagNoData = False
1295 1295
1296 1296 profileIndex = dataOut.profileIndex/n
1297 1297
1298 1298
1299 1299 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1300 1300
1301 1301 dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
1302 1302
1303 1303 dataOut.nProfiles = int(dataOut.nProfiles/n)
1304 1304
1305 1305 dataOut.profileIndex = profileIndex
1306 1306
1307 1307 dataOut.ippSeconds *= n
1308 1308
1309 1309 return dataOut
1310 1310
1311 1311 class PulsePairVoltage(Operation):
1312 1312 '''
1313 1313 Function PulsePair(Signal Power, Velocity)
1314 1314 The real component of Lag[0] provides Intensity Information
1315 1315 The imag component of Lag[1] Phase provides Velocity Information
1316 1316
1317 1317 Configuration Parameters:
1318 1318 nPRF = Number of Several PRF
1319 1319 theta = Degree Azimuth angel Boundaries
1320 1320
1321 1321 Input:
1322 1322 self.dataOut
1323 1323 lag[N]
1324 1324 Affected:
1325 1325 self.dataOut.spc
1326 1326 '''
1327 1327 isConfig = False
1328 1328 __profIndex = 0
1329 1329 __initime = None
1330 1330 __lastdatatime = None
1331 1331 __buffer = None
1332 1332 noise = None
1333 1333 __dataReady = False
1334 1334 n = None
1335 1335 __nch = 0
1336 1336 __nHeis = 0
1337 1337 removeDC = False
1338 1338 ipp = None
1339 1339 lambda_ = 0
1340 1340
1341 1341 def __init__(self,**kwargs):
1342 1342 Operation.__init__(self,**kwargs)
1343 1343
1344 1344 def setup(self, dataOut, n = None, removeDC=False):
1345 1345 '''
1346 1346 n= Numero de PRF's de entrada
1347 1347 '''
1348 1348 self.__initime = None
1349 1349 self.__lastdatatime = 0
1350 1350 self.__dataReady = False
1351 1351 self.__buffer = 0
1352 1352 self.__profIndex = 0
1353 1353 self.noise = None
1354 1354 self.__nch = dataOut.nChannels
1355 1355 self.__nHeis = dataOut.nHeights
1356 1356 self.removeDC = removeDC
1357 1357 self.lambda_ = 3.0e8/(9345.0e6)
1358 1358 self.ippSec = dataOut.ippSeconds
1359 1359 self.nCohInt = dataOut.nCohInt
1360 1360
1361 1361 if n == None:
1362 1362 raise ValueError("n should be specified.")
1363 1363
1364 1364 if n != None:
1365 1365 if n<2:
1366 1366 raise ValueError("n should be greater than 2")
1367 1367
1368 1368 self.n = n
1369 1369 self.__nProf = n
1370 1370
1371 1371 self.__buffer = numpy.zeros((dataOut.nChannels,
1372 1372 n,
1373 1373 dataOut.nHeights),
1374 1374 dtype='complex')
1375 1375
1376 1376 def putData(self,data):
1377 1377 '''
1378 1378 Add a profile to he __buffer and increase in one the __profiel Index
1379 1379 '''
1380 1380 self.__buffer[:,self.__profIndex,:]= data
1381 1381 self.__profIndex += 1
1382 1382 return
1383 1383
1384 1384 def pushData(self,dataOut):
1385 1385 '''
1386 1386 Return the PULSEPAIR and the profiles used in the operation
1387 1387 Affected : self.__profileIndex
1388 1388 '''
1389 1389 #----------------- Remove DC-----------------------------------
1390 1390 if self.removeDC==True:
1391 1391 mean = numpy.mean(self.__buffer,1)
1392 1392 tmp = mean.reshape(self.__nch,1,self.__nHeis)
1393 1393 dc= numpy.tile(tmp,[1,self.__nProf,1])
1394 1394 self.__buffer = self.__buffer - dc
1395 1395 #------------------Calculo de Potencia ------------------------
1396 1396 pair0 = self.__buffer*numpy.conj(self.__buffer)
1397 1397 pair0 = pair0.real
1398 1398 lag_0 = numpy.sum(pair0,1)
1399 1399 #------------------Calculo de Ruido x canal--------------------
1400 1400 self.noise = numpy.zeros(self.__nch)
1401 1401 for i in range(self.__nch):
1402 1402 daux = numpy.sort(pair0[i,:,:],axis= None)
1403 1403 self.noise[i]=hildebrand_sekhon( daux ,self.nCohInt)
1404 1404
1405 1405 self.noise = self.noise.reshape(self.__nch,1)
1406 1406 self.noise = numpy.tile(self.noise,[1,self.__nHeis])
1407 1407 noise_buffer = self.noise.reshape(self.__nch,1,self.__nHeis)
1408 1408 noise_buffer = numpy.tile(noise_buffer,[1,self.__nProf,1])
1409 1409 #------------------ Potencia recibida= P , Potencia senal = S , Ruido= N--
1410 1410 #------------------ P= S+N ,P=lag_0/N ---------------------------------
1411 1411 #-------------------- Power --------------------------------------------------
1412 1412 data_power = lag_0/(self.n*self.nCohInt)
1413 1413 #------------------ Senal ---------------------------------------------------
1414 1414 data_intensity = pair0 - noise_buffer
1415 1415 data_intensity = numpy.sum(data_intensity,axis=1)*(self.n*self.nCohInt)#*self.nCohInt)
1416 1416 #data_intensity = (lag_0-self.noise*self.n)*(self.n*self.nCohInt)
1417 1417 for i in range(self.__nch):
1418 1418 for j in range(self.__nHeis):
1419 1419 if data_intensity[i][j] < 0:
1420 1420 data_intensity[i][j] = numpy.min(numpy.absolute(data_intensity[i][j]))
1421 1421
1422 1422 #----------------- Calculo de Frecuencia y Velocidad doppler--------
1423 1423 pair1 = self.__buffer[:,:-1,:]*numpy.conjugate(self.__buffer[:,1:,:])
1424 1424 lag_1 = numpy.sum(pair1,1)
1425 1425 data_freq = (-1/(2.0*math.pi*self.ippSec*self.nCohInt))*numpy.angle(lag_1)
1426 1426 data_velocity = (self.lambda_/2.0)*data_freq
1427 1427
1428 1428 #---------------- Potencia promedio estimada de la Senal-----------
1429 1429 lag_0 = lag_0/self.n
1430 1430 S = lag_0-self.noise
1431 1431
1432 1432 #---------------- Frecuencia Doppler promedio ---------------------
1433 1433 lag_1 = lag_1/(self.n-1)
1434 1434 R1 = numpy.abs(lag_1)
1435 1435
1436 1436 #---------------- Calculo del SNR----------------------------------
1437 1437 data_snrPP = S/self.noise
1438 1438 for i in range(self.__nch):
1439 1439 for j in range(self.__nHeis):
1440 1440 if data_snrPP[i][j] < 1.e-20:
1441 1441 data_snrPP[i][j] = 1.e-20
1442 1442
1443 1443 #----------------- Calculo del ancho espectral ----------------------
1444 1444 L = S/R1
1445 1445 L = numpy.where(L<0,1,L)
1446 1446 L = numpy.log(L)
1447 1447 tmp = numpy.sqrt(numpy.absolute(L))
1448 1448 data_specwidth = (self.lambda_/(2*math.sqrt(2)*math.pi*self.ippSec*self.nCohInt))*tmp*numpy.sign(L)
1449 1449 n = self.__profIndex
1450 1450
1451 1451 self.__buffer = numpy.zeros((self.__nch, self.__nProf,self.__nHeis), dtype='complex')
1452 1452 self.__profIndex = 0
1453 1453 return data_power,data_intensity,data_velocity,data_snrPP,data_specwidth,n
1454 1454
1455 1455
1456 1456 def pulsePairbyProfiles(self,dataOut):
1457 1457
1458 1458 self.__dataReady = False
1459 1459 data_power = None
1460 1460 data_intensity = None
1461 1461 data_velocity = None
1462 1462 data_specwidth = None
1463 1463 data_snrPP = None
1464 1464 self.putData(data=dataOut.data)
1465 1465 if self.__profIndex == self.n:
1466 1466 data_power,data_intensity, data_velocity,data_snrPP,data_specwidth, n = self.pushData(dataOut=dataOut)
1467 1467 self.__dataReady = True
1468 1468
1469 1469 return data_power, data_intensity, data_velocity, data_snrPP, data_specwidth
1470 1470
1471 1471
1472 1472 def pulsePairOp(self, dataOut, datatime= None):
1473 1473
1474 1474 if self.__initime == None:
1475 1475 self.__initime = datatime
1476 1476 data_power, data_intensity, data_velocity, data_snrPP, data_specwidth = self.pulsePairbyProfiles(dataOut)
1477 1477 self.__lastdatatime = datatime
1478 1478
1479 1479 if data_power is None:
1480 1480 return None, None, None,None,None,None
1481 1481
1482 1482 avgdatatime = self.__initime
1483 1483 deltatime = datatime - self.__lastdatatime
1484 1484 self.__initime = datatime
1485 1485
1486 1486 return data_power, data_intensity, data_velocity, data_snrPP, data_specwidth, avgdatatime
1487 1487
1488 1488 def run(self, dataOut,n = None,removeDC= False, overlapping= False,**kwargs):
1489 1489
1490 1490 if not self.isConfig:
1491 1491 self.setup(dataOut = dataOut, n = n , removeDC=removeDC , **kwargs)
1492 1492 self.isConfig = True
1493 1493 data_power, data_intensity, data_velocity,data_snrPP,data_specwidth, avgdatatime = self.pulsePairOp(dataOut, dataOut.utctime)
1494 1494 dataOut.flagNoData = True
1495 1495
1496 1496 if self.__dataReady:
1497 1497 dataOut.nCohInt *= self.n
1498 1498 dataOut.dataPP_POW = data_intensity # S
1499 1499 dataOut.dataPP_POWER = data_power # P
1500 1500 dataOut.dataPP_DOP = data_velocity
1501 1501 dataOut.dataPP_SNR = data_snrPP
1502 1502 dataOut.dataPP_WIDTH = data_specwidth
1503 1503 dataOut.PRFbyAngle = self.n #numero de PRF*cada angulo rotado que equivale a un tiempo.
1504 1504 dataOut.utctime = avgdatatime
1505 1505 dataOut.flagNoData = False
1506 1506 return dataOut
1507 1507
1508 1508
1509 1509
1510 1510 # import collections
1511 1511 # from scipy.stats import mode
1512 1512 #
1513 1513 # class Synchronize(Operation):
1514 1514 #
1515 1515 # isConfig = False
1516 1516 # __profIndex = 0
1517 1517 #
1518 1518 # def __init__(self, **kwargs):
1519 1519 #
1520 1520 # Operation.__init__(self, **kwargs)
1521 1521 # # self.isConfig = False
1522 1522 # self.__powBuffer = None
1523 1523 # self.__startIndex = 0
1524 1524 # self.__pulseFound = False
1525 1525 #
1526 1526 # def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
1527 1527 #
1528 1528 # #Read data
1529 1529 #
1530 1530 # powerdB = dataOut.getPower(channel = channel)
1531 1531 # noisedB = dataOut.getNoise(channel = channel)[0]
1532 1532 #
1533 1533 # self.__powBuffer.extend(powerdB.flatten())
1534 1534 #
1535 1535 # dataArray = numpy.array(self.__powBuffer)
1536 1536 #
1537 1537 # filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
1538 1538 #
1539 1539 # maxValue = numpy.nanmax(filteredPower)
1540 1540 #
1541 1541 # if maxValue < noisedB + 10:
1542 1542 # #No se encuentra ningun pulso de transmision
1543 1543 # return None
1544 1544 #
1545 1545 # maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
1546 1546 #
1547 1547 # if len(maxValuesIndex) < 2:
1548 1548 # #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
1549 1549 # return None
1550 1550 #
1551 1551 # phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
1552 1552 #
1553 1553 # #Seleccionar solo valores con un espaciamiento de nSamples
1554 1554 # pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
1555 1555 #
1556 1556 # if len(pulseIndex) < 2:
1557 1557 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1558 1558 # return None
1559 1559 #
1560 1560 # spacing = pulseIndex[1:] - pulseIndex[:-1]
1561 1561 #
1562 1562 # #remover senales que se distancien menos de 10 unidades o muestras
1563 1563 # #(No deberian existir IPP menor a 10 unidades)
1564 1564 #
1565 1565 # realIndex = numpy.where(spacing > 10 )[0]
1566 1566 #
1567 1567 # if len(realIndex) < 2:
1568 1568 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1569 1569 # return None
1570 1570 #
1571 1571 # #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
1572 1572 # realPulseIndex = pulseIndex[realIndex]
1573 1573 #
1574 1574 # period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
1575 1575 #
1576 1576 # print "IPP = %d samples" %period
1577 1577 #
1578 1578 # self.__newNSamples = dataOut.nHeights #int(period)
1579 1579 # self.__startIndex = int(realPulseIndex[0])
1580 1580 #
1581 1581 # return 1
1582 1582 #
1583 1583 #
1584 1584 # def setup(self, nSamples, nChannels, buffer_size = 4):
1585 1585 #
1586 1586 # self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
1587 1587 # maxlen = buffer_size*nSamples)
1588 1588 #
1589 1589 # bufferList = []
1590 1590 #
1591 1591 # for i in range(nChannels):
1592 1592 # bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) + numpy.NAN,
1593 1593 # maxlen = buffer_size*nSamples)
1594 1594 #
1595 1595 # bufferList.append(bufferByChannel)
1596 1596 #
1597 1597 # self.__nSamples = nSamples
1598 1598 # self.__nChannels = nChannels
1599 1599 # self.__bufferList = bufferList
1600 1600 #
1601 1601 # def run(self, dataOut, channel = 0):
1602 1602 #
1603 1603 # if not self.isConfig:
1604 1604 # nSamples = dataOut.nHeights
1605 1605 # nChannels = dataOut.nChannels
1606 1606 # self.setup(nSamples, nChannels)
1607 1607 # self.isConfig = True
1608 1608 #
1609 1609 # #Append new data to internal buffer
1610 1610 # for thisChannel in range(self.__nChannels):
1611 1611 # bufferByChannel = self.__bufferList[thisChannel]
1612 1612 # bufferByChannel.extend(dataOut.data[thisChannel])
1613 1613 #
1614 1614 # if self.__pulseFound:
1615 1615 # self.__startIndex -= self.__nSamples
1616 1616 #
1617 1617 # #Finding Tx Pulse
1618 1618 # if not self.__pulseFound:
1619 1619 # indexFound = self.__findTxPulse(dataOut, channel)
1620 1620 #
1621 1621 # if indexFound == None:
1622 1622 # dataOut.flagNoData = True
1623 1623 # return
1624 1624 #
1625 1625 # self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
1626 1626 # self.__pulseFound = True
1627 1627 # self.__startIndex = indexFound
1628 1628 #
1629 1629 # #If pulse was found ...
1630 1630 # for thisChannel in range(self.__nChannels):
1631 1631 # bufferByChannel = self.__bufferList[thisChannel]
1632 1632 # #print self.__startIndex
1633 1633 # x = numpy.array(bufferByChannel)
1634 1634 # self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
1635 1635 #
1636 1636 # deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1637 1637 # dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
1638 1638 # # dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
1639 1639 #
1640 1640 # dataOut.data = self.__arrayBuffer
1641 1641 #
1642 1642 # self.__startIndex += self.__newNSamples
1643 1643 #
1644 1644 # return
1645 1645 class SSheightProfiles(Operation):
1646 1646
1647 1647 step = None
1648 1648 nsamples = None
1649 1649 bufferShape = None
1650 1650 profileShape = None
1651 1651 sshProfiles = None
1652 1652 profileIndex = None
1653 1653
1654 1654 def __init__(self, **kwargs):
1655 1655
1656 1656 Operation.__init__(self, **kwargs)
1657 1657 self.isConfig = False
1658 1658
1659 1659 def setup(self,dataOut ,step = None , nsamples = None):
1660 1660
1661 1661 if step == None and nsamples == None:
1662 1662 raise ValueError("step or nheights should be specified ...")
1663 1663
1664 1664 self.step = step
1665 1665 self.nsamples = nsamples
1666 1666 self.__nChannels = dataOut.nChannels
1667 1667 self.__nProfiles = dataOut.nProfiles
1668 1668 self.__nHeis = dataOut.nHeights
1669 1669 shape = dataOut.data.shape #nchannels, nprofiles, nsamples
1670 1670
1671 1671 residue = (shape[1] - self.nsamples) % self.step
1672 1672 if residue != 0:
1673 1673 print("The residue is %d, step=%d should be multiple of %d to avoid loss of %d samples"%(residue,step,shape[1] - self.nsamples,residue))
1674 1674
1675 1675 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1676 1676 numberProfile = self.nsamples
1677 1677 numberSamples = (shape[1] - self.nsamples)/self.step
1678 1678
1679 1679 self.bufferShape = int(shape[0]), int(numberSamples), int(numberProfile) # nchannels, nsamples , nprofiles
1680 1680 self.profileShape = int(shape[0]), int(numberProfile), int(numberSamples) # nchannels, nprofiles, nsamples
1681 1681
1682 1682 self.buffer = numpy.zeros(self.bufferShape , dtype=numpy.complex)
1683 1683 self.sshProfiles = numpy.zeros(self.profileShape, dtype=numpy.complex)
1684 1684
1685 1685 def run(self, dataOut, step, nsamples, code = None, repeat = None):
1686 1686 dataOut.flagNoData = True
1687 1687
1688 1688 profileIndex = None
1689 1689 #print("nProfiles, nHeights ",dataOut.nProfiles, dataOut.nHeights)
1690 1690 #print(dataOut.getFreqRange(1)/1000.)
1691 1691 #exit(1)
1692 1692 if dataOut.flagDataAsBlock:
1693 1693 dataOut.data = numpy.average(dataOut.data,axis=1)
1694 1694 #print("jee")
1695 1695 dataOut.flagDataAsBlock = False
1696 1696 if not self.isConfig:
1697 1697 self.setup(dataOut, step=step , nsamples=nsamples)
1698 1698 #print("Setup done")
1699 1699 self.isConfig = True
1700 1700
1701 1701
1702 1702 if code is not None:
1703 1703 code = numpy.array(code)
1704 1704 code_block = code
1705 1705
1706 1706 if repeat is not None:
1707 1707 code_block = numpy.repeat(code_block, repeats=repeat, axis=1)
1708 1708 #print(code_block.shape)
1709 1709 for i in range(self.buffer.shape[1]):
1710 1710
1711 1711 if code is not None:
1712 1712 self.buffer[:,i] = dataOut.data[:,i*self.step:i*self.step + self.nsamples]*code_block
1713 1713
1714 1714 else:
1715 1715
1716 1716 self.buffer[:,i] = dataOut.data[:,i*self.step:i*self.step + self.nsamples]#*code[dataOut.profileIndex,:]
1717 1717
1718 1718 #self.buffer[:,j,self.__nHeis-j*self.step - self.nheights:self.__nHeis-j*self.step] = numpy.flip(dataOut.data[:,j*self.step:j*self.step + self.nheights])
1719 1719
1720 1720 for j in range(self.buffer.shape[0]):
1721 1721 self.sshProfiles[j] = numpy.transpose(self.buffer[j])
1722 1722
1723 1723 profileIndex = self.nsamples
1724 1724 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1725 1725 ippSeconds = (deltaHeight*1.0e-6)/(0.15)
1726 1726 #print("ippSeconds, dH: ",ippSeconds,deltaHeight)
1727 1727 try:
1728 1728 if dataOut.concat_m is not None:
1729 1729 ippSeconds= ippSeconds/float(dataOut.concat_m)
1730 1730 #print "Profile concat %d"%dataOut.concat_m
1731 1731 except:
1732 1732 pass
1733 1733
1734 1734 dataOut.data = self.sshProfiles
1735 1735 dataOut.flagNoData = False
1736 1736 dataOut.heightList = numpy.arange(self.buffer.shape[1]) *self.step*deltaHeight + dataOut.heightList[0]
1737 1737 dataOut.nProfiles = int(dataOut.nProfiles*self.nsamples)
1738 1738
1739 1739 dataOut.profileIndex = profileIndex
1740 1740 dataOut.flagDataAsBlock = True
1741 1741 dataOut.ippSeconds = ippSeconds
1742 1742 dataOut.step = self.step
1743 1743 #print(numpy.shape(dataOut.data))
1744 1744 #exit(1)
1745 #print("new data shape and time:", dataOut.data.shape, dataOut.utctime)
1745 1746
1746 1747 return dataOut
1747 1748 ################################################################################3############################3
1748 1749 ################################################################################3############################3
1749 1750 ################################################################################3############################3
1750 1751 ################################################################################3############################3
1751 1752
1752 1753 class SSheightProfiles2(Operation):
1753 1754 '''
1754 1755 Procesa por perfiles y por bloques
1755 1756 '''
1756 1757
1757 step = None
1758 nsamples = None
1758
1759 1759 bufferShape = None
1760 1760 profileShape = None
1761 1761 sshProfiles = None
1762 1762 profileIndex = None
1763 deltaHeight = None
1764 init_range = None
1763 #nsamples = None
1764 #step = None
1765 #deltaHeight = None
1766 #init_range = None
1767 __slots__ = ('step', 'nsamples', 'deltaHeight', 'init_range', 'isConfig', '__nChannels',
1768 '__nProfiles', '__nHeis', 'deltaHeight', 'new_nHeights')
1765 1769
1766 1770 def __init__(self, **kwargs):
1767 1771
1768 1772 Operation.__init__(self, **kwargs)
1769 1773 self.isConfig = False
1770 1774
1771 1775 def setup(self,dataOut ,step = None , nsamples = None):
1772 1776
1773 1777 if step == None and nsamples == None:
1774 1778 raise ValueError("step or nheights should be specified ...")
1775 1779
1776 1780 self.step = step
1777 1781 self.nsamples = nsamples
1778 1782 self.__nChannels = int(dataOut.nChannels)
1779 1783 self.__nProfiles = int(dataOut.nProfiles)
1780 1784 self.__nHeis = int(dataOut.nHeights)
1781 1785
1782 1786 residue = (self.__nHeis - self.nsamples) % self.step
1783 1787 if residue != 0:
1784 1788 print("The residue is %d, step=%d should be multiple of %d to avoid loss of %d samples"%(residue,step,shape[1] - self.nsamples,residue))
1785 1789
1786 1790 self.deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1787 1791 self.init_range = dataOut.heightList[0]
1788 1792 #numberProfile = self.nsamples
1789 self.numberSamples = (self.__nHeis - self.nsamples)/self.step
1793 numberSamples = (self.__nHeis - self.nsamples)/self.step
1790 1794
1791 self.new_nHeights = self.numberSamples
1795 self.new_nHeights = numberSamples
1792 1796
1793 self.bufferShape = int(self.__nChannels), int(self.numberSamples), int(self.nsamples) # nchannels, nsamples , nprofiles
1794 self.profileShape = int(self.__nChannels), int(self.nsamples), int(self.numberSamples) # nchannels, nprofiles, nsamples
1797 self.bufferShape = int(self.__nChannels), int(numberSamples), int(self.nsamples) # nchannels, nsamples , nprofiles
1798 self.profileShape = int(self.__nChannels), int(self.nsamples), int(numberSamples) # nchannels, nprofiles, nsamples
1795 1799
1796 1800 self.buffer = numpy.zeros(self.bufferShape , dtype=numpy.complex)
1797 1801 self.sshProfiles = numpy.zeros(self.profileShape, dtype=numpy.complex)
1798 1802
1799 1803 def getNewProfiles(self, data, code=None, repeat=None):
1800 1804
1801 1805 if code is not None:
1802 1806 code = numpy.array(code)
1803 1807 code_block = code
1804 1808
1805 1809 if repeat is not None:
1806 1810 code_block = numpy.repeat(code_block, repeats=repeat, axis=1)
1807 1811 if data.ndim == 2:
1808 1812 data = data.reshape(1,1,self.__nHeis )
1809 1813 #print("buff, data, :",self.buffer.shape, data.shape,self.sshProfiles.shape)
1810 1814 for i in range(int(self.new_nHeights)): #nuevas alturas
1811 1815 if code is not None:
1812 1816 self.buffer[:,i,:] = data[:,:,i*self.step:i*self.step + self.nsamples]*code_block
1813 1817 else:
1814 1818 self.buffer[:,i,:] = data[:,:,i*self.step:i*self.step + self.nsamples]#*code[dataOut.profileIndex,:]
1815 1819
1816 1820 for j in range(self.__nChannels): #en los cananles
1817 1821 self.sshProfiles[j,:,:] = numpy.transpose(self.buffer[j,:,:])
1818 1822 #print("new profs Done")
1819 1823
1820 1824
1821 1825
1822 1826 def run(self, dataOut, step, nsamples, code = None, repeat = None):
1823 1827
1824 1828 if dataOut.flagNoData == True:
1825 1829 return dataOut
1826 1830 dataOut.flagNoData = True
1827 1831 #print("init data shape:", dataOut.data.shape)
1828 1832 #print("ch: {} prof: {} hs: {}".format(int(dataOut.nChannels),
1829 1833 # int(dataOut.nProfiles),int(dataOut.nHeights)))
1830 1834
1831 1835 profileIndex = None
1832 1836 # if not dataOut.flagDataAsBlock:
1833 1837 # dataOut.nProfiles = 1
1834 1838
1835 1839 if not self.isConfig:
1836 1840 self.setup(dataOut, step=step , nsamples=nsamples)
1837 1841 #print("Setup done")
1838 1842 self.isConfig = True
1839 1843
1840 1844 dataBlock = None
1841 1845
1842 1846 nprof = 1
1843 1847 if dataOut.flagDataAsBlock:
1844 1848 nprof = int(dataOut.nProfiles)
1845 1849
1846 1850 #print("dataOut nProfiles:", dataOut.nProfiles)
1847 1851 for profile in range(nprof):
1848 1852 if dataOut.flagDataAsBlock:
1849 1853 #print("read blocks")
1850 1854 self.getNewProfiles(dataOut.data[:,profile,:], code=code, repeat=repeat)
1851 1855 else:
1852 1856 #print("read profiles")
1853 1857 self.getNewProfiles(dataOut.data, code=code, repeat=repeat) #only one channe
1854 1858 if profile == 0:
1855 1859 dataBlock = self.sshProfiles.copy()
1856 1860 else: #by blocks
1857 1861 dataBlock = numpy.concatenate((dataBlock,self.sshProfiles), axis=1) #profile axis
1858 1862 #print("by blocks: ",dataBlock.shape, self.sshProfiles.shape)
1859 1863
1860 1864 profileIndex = self.nsamples
1861 1865 #deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1862 1866 ippSeconds = (self.deltaHeight*1.0e-6)/(0.15)
1863 1867
1864 1868
1865 1869 dataOut.data = dataBlock
1866 1870 #print("show me: ",self.step,self.deltaHeight, dataOut.heightList, self.new_nHeights)
1867 1871 dataOut.heightList = numpy.arange(int(self.new_nHeights)) *self.step*self.deltaHeight + self.init_range
1868 1872
1869 1873 dataOut.ippSeconds = ippSeconds
1870 1874 dataOut.step = self.step
1871 1875 dataOut.flagNoData = False
1872 1876 if dataOut.flagDataAsBlock:
1873 1877 dataOut.nProfiles = int(dataOut.nProfiles*self.nsamples)
1874 1878
1875 1879 else:
1876 1880 dataOut.nProfiles = int(self.nsamples)
1877 1881 dataOut.profileIndex = dataOut.nProfiles
1878 1882 dataOut.flagDataAsBlock = True
1879 1883
1880 1884 dataBlock = None
1881 #print("new data shape:", dataOut.data.shape)
1885
1886 #print("new data shape:", dataOut.data.shape, dataOut.utctime)
1882 1887
1883 1888 return dataOut
1884 1889
1885 1890
1886 1891
1887 1892
1888 1893 #import skimage.color
1889 1894 #import skimage.io
1890 1895 #import matplotlib.pyplot as plt
1891 1896
1892 1897 class removeProfileByFaradayHS(Operation):
1893 1898 '''
1894 1899
1895 1900 '''
1896 isConfig = False
1897 n = None
1898
1899 __profIndex = 0
1901 #isConfig = False
1902 #n = None
1900 1903
1901 __dataReady = False
1904 #__dataReady = False
1902 1905 __buffer_data = []
1903 1906 __buffer_times = []
1904 __initime = None
1905 __count_exec = 0
1906 __profIndex = 0
1907 #__initime = None
1908 #__count_exec = 0
1909 #__profIndex = 0
1907 1910 buffer = None
1908 lenProfileOut = 1
1911 #lenProfileOut = 1
1912
1913 #init_prof = 0
1914 #end_prof = 0
1909 1915
1910 init_prof = 0
1911 end_prof = 0
1912 n_profiles = 0
1913 first_utcBlock = None
1916 #first_utcBlock = None
1914 1917 outliers_IDs_list = []
1915 __dh = 0
1918 #__dh = 0
1916 1919
1920 __slots__ = ('n','navg','profileMargin','thHistOutlier','minHei_idx','maxHei_idx','nHeights',
1921 '__dh','first_utcBlock','__profIndex','init_prof','end_prof','lenProfileOut','nChannels',
1922 '__count_exec','__initime','__dataReady','__ipp')
1917 1923 def __init__(self, **kwargs):
1918 1924
1919 1925 Operation.__init__(self, **kwargs)
1920 1926 self.isConfig = False
1921 1927
1922 1928 def setup(self,dataOut, n=None , navg=0.8, profileMargin=50,thHistOutlier=3, minHei=None, maxHei=None):
1923 1929
1924 1930 if n == None and timeInterval == None:
1925 1931 raise ValueError("nprofiles or timeInterval should be specified ...")
1926 1932
1927 1933 if n != None:
1928 1934 self.n = n
1929 1935
1930 1936 self.navg = navg
1931 1937 self.profileMargin = profileMargin
1932 1938 self.thHistOutlier = thHistOutlier
1933 1939 self.__profIndex = 0
1934 1940 self.buffer = None
1935 self.lenProfileOut = 1
1941 self._ipp = dataOut.ippSeconds
1936 1942 self.n_prof_released = 0
1937 1943 self.heightList = dataOut.heightList
1938 1944 self.init_prof = 0
1939 1945 self.end_prof = 0
1940 self.n_profiles = 0
1946 self.__count_exec = 0
1947 self.__profIndex = 0
1941 1948 self.first_utcBlock = None
1942 1949 self.__dh = dataOut.heightList[1] - dataOut.heightList[0]
1943 1950 minHei = minHei
1944 1951 maxHei = maxHei
1945 1952 if minHei==None :
1946 1953 minHei = dataOut.heightList[0]
1947 1954 if maxHei==None :
1948 1955 maxHei = dataOut.heightList[-1]
1949 1956 self.minHei_idx,self.maxHei_idx = getHei_index(minHei, maxHei, dataOut.heightList)
1950 1957
1958 self.nChannels = dataOut.nChannels
1959 self.nHeights = dataOut.nHeights
1951 1960
1952 1961 def filterSatsProfiles(self):
1953 1962 data = self.__buffer_data
1954 1963 #print(data.shape)
1955 1964 nChannels, profiles, heights = data.shape
1956 1965 indexes=[]
1957 1966 outliers_IDs=[]
1958 1967 for c in range(nChannels):
1959 1968 for h in range(self.minHei_idx, self.maxHei_idx):
1960 1969 power = data[c,:,h] * numpy.conjugate(data[c,:,h])
1961 1970 power = power.real
1962 1971 #power = (numpy.abs(data[c,:,h].real))
1963 1972 sortdata = numpy.sort(power, axis=None)
1964 1973 sortID=power.argsort()
1965 1974 index = _noise.hildebrand_sekhon2(sortdata,self.navg) #0.75-> buen valor
1966 1975
1967 1976 indexes.append(index)
1968 1977 outliers_IDs=numpy.append(outliers_IDs,sortID[index:])
1969 1978 # print(outliers_IDs)
1970 1979 # fig,ax = plt.subplots()
1971 1980 # #ax.set_title(str(k)+" "+str(j))
1972 1981 # x=range(len(sortdata))
1973 1982 # ax.scatter(x,sortdata)
1974 1983 # ax.axvline(index)
1975 1984 # plt.grid()
1976 1985 # plt.show()
1977 1986
1978 1987 outliers_IDs = outliers_IDs.astype(numpy.dtype('int64'))
1979 1988 outliers_IDs = numpy.unique(outliers_IDs)
1980 1989 outs_lines = numpy.sort(outliers_IDs)
1981 1990 # #print("outliers Ids: ", outs_lines, outs_lines.shape)
1982 1991 #hist, bin_edges = numpy.histogram(outs_lines, bins=10, density=True)
1983 1992
1984 1993
1985 1994 #Agrupando el histograma de outliers,
1986 1995 my_bins = numpy.linspace(0,9600, 96, endpoint=False)
1987 1996
1988 1997
1989 1998 hist, bins = numpy.histogram(outs_lines,bins=my_bins)
1990 1999 hist_outliers_indexes = numpy.where(hist > self.thHistOutlier) #es outlier
1991 2000 #print(hist_outliers_indexes[0])
1992 2001 bins_outliers_indexes = [int(i) for i in bins[hist_outliers_indexes]] #
1993 2002 #print(bins_outliers_indexes)
1994 2003 outlier_loc_index = []
1995 2004
1996 2005 #outlier_loc_index = [k for k in range(bins_outliers_indexes[n]-50,bins_outliers_indexes[n+1]+50) for n in range(len(bins_outliers_indexes)-1) ]
1997 2006 for n in range(len(bins_outliers_indexes)-1):
1998 2007 #outlier_loc_index = [k for k in range(bins_outliers_indexes[n]-50,bins_outliers_indexes[n+1]+50)]
1999 2008 for k in range(bins_outliers_indexes[n]-self.profileMargin,bins_outliers_indexes[n+1]+self.profileMargin):
2000 2009 outlier_loc_index.append(k)
2001 2010
2002 2011 outlier_loc_index = numpy.asarray(outlier_loc_index)
2003 2012 #print(numpy.unique(outlier_loc_index))
2004 2013
2005 2014
2006 2015
2007 2016 # x, y = numpy.meshgrid(numpy.arange(profiles), self.heightList)
2008 2017 # fig, ax = plt.subplots(1,2,figsize=(8, 6))
2009 2018 #
2010 2019 # dat = data[0,:,:].real
2011 2020 # m = numpy.nanmean(dat)
2012 2021 # o = numpy.nanstd(dat)
2013 2022 # #print(m, o, x.shape, y.shape)
2014 2023 # c = ax[0].pcolormesh(x, y, dat.T, cmap ='YlGnBu', vmin = (m-2*o), vmax = (m+2*o))
2015 2024 # ax[0].vlines(outs_lines,200,600, linestyles='dashed', label = 'outs', color='w')
2016 2025 # fig.colorbar(c)
2017 2026 # ax[0].vlines(outlier_loc_index,650,750, linestyles='dashed', label = 'outs', color='r')
2018 2027 # ax[1].hist(outs_lines,bins=my_bins)
2019 2028 # plt.show()
2020 2029
2021 2030
2022 2031 self.outliers_IDs_list = numpy.unique(outlier_loc_index)
2023 2032 return data
2024 2033
2025 2034 def cleanOutliersByBlock(self):
2026 2035 #print(self.__buffer_data[0].shape)
2027 2036 data = self.__buffer_data#.copy()
2028 2037 #print("cleaning shape inpt: ",data.shape)
2029 2038 '''
2030 2039 self.__buffer_data = []
2031 2040
2032 2041 spectrum = numpy.fft.fft2(data, axes=(0,2))
2033 2042 #print("spc : ",spectrum.shape)
2034 2043 (nch,nsamples, nh) = spectrum.shape
2035 2044 data2 = None
2036 2045 #print(data.shape)
2037 2046 spectrum2 = spectrum.copy()
2038 2047 for ch in range(nch):
2039 2048 dh = self.__dh
2040 2049 dt1 = (dh*1.0e-6)/(0.15)
2041 2050 dt2 = self.__buffer_times[1]-self.__buffer_times[0]
2042 2051 freqv = numpy.fft.fftfreq(nh, d=dt1)
2043 2052 freqh = numpy.fft.fftfreq(self.n, d=dt2)
2044 2053 #print("spc loop: ")
2045 2054
2046 2055
2047 2056
2048 2057 x, y = numpy.meshgrid(numpy.sort(freqh),numpy.sort(freqv))
2049 2058 z = numpy.abs(spectrum[ch,:,:])
2050 2059 # Find all peaks higher than the 98th percentile
2051 2060 peaks = z < numpy.percentile(z, 98)
2052 2061 #print(peaks)
2053 2062 # Set those peak coefficients to zero
2054 2063 spectrum2 = spectrum2 * peaks.astype(int)
2055 2064 data2 = numpy.fft.ifft2(spectrum2)
2056 2065
2057 2066 dat = numpy.log10(z.T)
2058 2067 dat2 = numpy.log10(spectrum2.T)
2059 2068
2060 2069 # m = numpy.mean(dat)
2061 2070 # o = numpy.std(dat)
2062 2071 # fig, ax = plt.subplots(2,1,figsize=(8, 6))
2063 2072 #
2064 2073 # c = ax[0].pcolormesh(x, y, dat, cmap ='YlGnBu', vmin = (m-2*o), vmax = (m+2*o))
2065 2074 # #c = ax.pcolor( z.T , cmap ='gray', vmin = (m-2*o), vmax = (m+2*o))
2066 2075 # date_time = datetime.datetime.fromtimestamp(self.__buffer_times[0]).strftime('%Y-%m-%d %H:%M:%S.%f')
2067 2076 # #strftime('%Y-%m-%d %H:%M:%S')
2068 2077 # ax[0].set_title('Spectrum magnitude '+date_time)
2069 2078 # fig.canvas.set_window_title('Spectrum magnitude {} '.format(self.n)+date_time)
2070 2079 #
2071 2080 #
2072 2081 # c = ax[1].pcolormesh(x, y, dat, cmap ='YlGnBu', vmin = (m-2*o), vmax = (m+2*o))
2073 2082 # fig.colorbar(c)
2074 2083 # plt.show()
2075 2084
2076 2085 #print(data2.shape)
2077 2086
2078 2087 data = data2
2079 2088
2080 2089 #cleanBlock = numpy.fft.ifft2(spectrum, axes=(0,2)).reshape()
2081 2090 '''
2082 2091 #print("cleanOutliersByBlock Done")
2083 2092
2084 2093 return self.filterSatsProfiles()
2085 2094
2086 2095
2087 2096
2088 2097 def fillBuffer(self, data, datatime):
2089 2098
2090 2099 if self.__profIndex == 0:
2091 2100 self.__buffer_data = data.copy()
2092 2101
2093 2102 else:
2094 2103 self.__buffer_data = numpy.concatenate((self.__buffer_data,data), axis=1)#en perfiles
2095 2104 self.__profIndex += 1
2096 2105 #self.__buffer_times.append(datatime)
2097 2106
2098 2107 def getData(self, data, datatime=None):
2099 2108
2100 2109 if self.__profIndex == 0:
2101 2110 self.__initime = datatime
2102 2111
2103 2112
2104 2113 self.__dataReady = False
2105 2114
2106 2115 self.fillBuffer(data, datatime)
2107 2116 dataBlock = None
2108 2117
2109 2118 if self.__profIndex == self.n:
2110 2119 #print("apnd : ",data)
2111 2120 #dataBlock = self.cleanOutliersByBlock()
2112 2121 dataBlock = self.filterSatsProfiles()
2113 2122 self.__dataReady = True
2114 2123
2115 2124 return dataBlock
2116 2125
2117 2126 if dataBlock is None:
2118 2127 return None, None
2119 2128
2120 2129
2121 2130
2122 2131 return dataBlock
2123 2132
2124 2133 def releaseBlock(self):
2125 2134
2126 2135 if self.n % self.lenProfileOut != 0:
2127 raise ValueError("lenProfileOut %d must be submultiple of nProfiles %d" %(self.lenProfileOut, self.n_profiles))
2136 raise ValueError("lenProfileOut %d must be submultiple of nProfiles %d" %(self.lenProfileOut, self.n))
2128 2137 return None
2129 2138
2130 2139 data = self.buffer[:,self.init_prof:self.end_prof:,:] #ch, prof, alt
2131 2140
2132 2141 self.init_prof = self.end_prof
2133 2142 self.end_prof += self.lenProfileOut
2134 2143 #print("data release shape: ",dataOut.data.shape, self.end_prof)
2135 2144 self.n_prof_released += 1
2136 2145
2137 2146
2138 2147 #print("f_no_data ", dataOut.flagNoData)
2139 2148 return data
2140 2149
2141 2150 def run(self, dataOut, n=None, navg=0.8, nProfilesOut=1, profile_margin=50,th_hist_outlier=3,minHei=None, maxHei=None):
2142 #print("run op buffer 2D")
2143 self.nChannels = dataOut.nChannels
2144 self.nHeights = dataOut.nHeights
2151 #print("run op buffer 2D",dataOut.ippSeconds)
2152 # self.nChannels = dataOut.nChannels
2153 # self.nHeights = dataOut.nHeights
2145 2154
2146 2155 if not self.isConfig:
2147 2156 #print("init p idx: ", dataOut.profileIndex )
2148 2157 self.setup(dataOut,n=n, navg=navg,profileMargin=profile_margin,
2149 2158 thHistOutlier=th_hist_outlier,minHei=minHei, maxHei=maxHei)
2150 2159 self.isConfig = True
2151 2160
2152 2161 dataBlock = None
2153 2162
2154 2163 if not dataOut.buffer_empty: #hay datos acumulados
2155 2164
2156 2165 if self.init_prof == 0:
2157 2166 self.n_prof_released = 0
2158 2167 self.lenProfileOut = nProfilesOut
2159 2168 dataOut.flagNoData = False
2160 2169 #print("tp 2 ",dataOut.data.shape)
2161 #(ch, self.n_profiles, nh) = self.buffer.shape
2162 #print("tp 3 ",self.dataOut.data.shape)
2163 #print("rel: ",self.buffer[:,-1,:])
2170
2164 2171 self.init_prof = 0
2165 2172 self.end_prof = self.lenProfileOut
2166 2173
2167 2174 dataOut.nProfiles = self.lenProfileOut
2168 2175 if nProfilesOut == 1:
2169 2176 dataOut.flagDataAsBlock = False
2170 2177 else:
2171 2178 dataOut.flagDataAsBlock = True
2172 2179 #print("prof: ",self.init_prof)
2173 2180 dataOut.flagNoData = False
2174 2181 if numpy.isin(self.n_prof_released, self.outliers_IDs_list):
2175 2182 #print("omitting: ", self.n_prof_released)
2176 2183 dataOut.flagNoData = True
2177
2178 dataOut.utctime = self.first_utcBlock + self.init_prof*dataOut.ippSeconds
2184 dataOut.ippSeconds = self._ipp
2185 dataOut.utctime = self.first_utcBlock + self.init_prof*self._ipp
2186 # print("time: ", dataOut.utctime, self.first_utcBlock, self.init_prof,self._ipp,dataOut.ippSeconds)
2179 2187 #dataOut.data = self.releaseBlock()
2180 2188 #########################################################3
2181 2189 if self.n % self.lenProfileOut != 0:
2182 raise ValueError("lenProfileOut %d must be submultiple of nProfiles %d" %(self.lenProfileOut, self.n_profiles))
2190 raise ValueError("lenProfileOut %d must be submultiple of nProfiles %d" %(self.lenProfileOut, self.n))
2183 2191 return None
2184 2192
2185 2193 dataOut.data = self.buffer[:,self.init_prof:self.end_prof:,:] #ch, prof, alt
2186 2194
2187 2195 self.init_prof = self.end_prof
2188 2196 self.end_prof += self.lenProfileOut
2189 2197 #print("data release shape: ",dataOut.data.shape, self.end_prof)
2190 2198 self.n_prof_released += 1
2191 2199
2192 2200 if self.end_prof >= (self.n +self.lenProfileOut):
2193 2201
2194 2202 self.init_prof = 0
2195 2203 self.__profIndex = 0
2196 2204 self.buffer = None
2197 2205 dataOut.buffer_empty = True
2198 2206 self.outliers_IDs_list = []
2199 2207 self.n_prof_released = 0
2200 2208 dataOut.flagNoData = False #enviar ultimo aunque sea outlier :(
2201 2209 #print("cleaning...", dataOut.buffer_empty)
2202 2210 dataOut.profileIndex = 0 #self.lenProfileOut
2203 2211 ####################################################################
2204 2212 return dataOut
2205 2213
2206 2214
2207 2215 #print("tp 223 ",dataOut.data.shape)
2208 2216 dataOut.flagNoData = True
2209 2217
2210 2218
2211 2219
2212 2220 try:
2213 2221 #dataBlock = self.getData(dataOut.data.reshape(self.nChannels,1,self.nHeights), dataOut.utctime)
2214 2222 dataBlock = self.getData(numpy.reshape(dataOut.data,(self.nChannels,1,self.nHeights)), dataOut.utctime)
2215 2223 self.__count_exec +=1
2216 2224 except Exception as e:
2217 2225 print("Error getting profiles data",self.__count_exec )
2218 2226 print(e)
2219 2227 sys.exit()
2220 2228
2221 2229 if self.__dataReady:
2222 2230 #print("omitting: ", len(self.outliers_IDs_list))
2223 2231 self.__count_exec = 0
2224 2232 #dataOut.data =
2225 2233 #self.buffer = numpy.flip(dataBlock, axis=1)
2226 2234 self.buffer = dataBlock
2227 2235 self.first_utcBlock = self.__initime
2228 2236 dataOut.utctime = self.__initime
2229 2237 dataOut.nProfiles = self.__profIndex
2230 2238 #dataOut.flagNoData = False
2231 2239 self.init_prof = 0
2232 2240 self.__profIndex = 0
2233 2241 self.__initime = None
2234 2242 dataBlock = None
2235 2243 self.__buffer_times = []
2236 2244 dataOut.error = False
2237 2245 dataOut.useInputBuffer = True
2238 2246 dataOut.buffer_empty = False
2239 2247 #print("1 ch: {} prof: {} hs: {}".format(int(dataOut.nChannels),int(dataOut.nProfiles),int(dataOut.nHeights)))
2240 2248
2241 2249
2242 2250
2243 2251 #print(self.__count_exec)
2244 2252
2245 2253 return dataOut
2246 2254
2247 2255 class RemoveProfileSats(Operation):
2248 2256 '''
2249 2257 Omite los perfiles contaminados con señal de satelites,
2250 2258 In: minHei = min_sat_range
2251 2259 max_sat_range
2252 2260 min_hei_ref
2253 2261 max_hei_ref
2254 2262 th = diference between profiles mean, ref and sats
2255 2263 Out:
2256 2264 profile clean
2257 2265 '''
2258 2266
2259 2267 isConfig = False
2260 2268 min_sats = 0
2261 2269 max_sats = 999999999
2262 2270 min_ref= 0
2263 2271 max_ref= 9999999999
2264 2272 needReshape = False
2265 2273 count = 0
2266 2274 thdB = 0
2267 2275 byRanges = False
2268 2276 min_sats = None
2269 2277 max_sats = None
2270 2278 noise = 0
2271 2279
2272 2280 def __init__(self, **kwargs):
2273 2281
2274 2282 Operation.__init__(self, **kwargs)
2275 2283 self.isConfig = False
2276 2284
2277 2285
2278 2286 def setup(self, dataOut, minHei, maxHei, minRef, maxRef, th, thdB, rangeHeiList):
2279 2287
2280 2288 if rangeHeiList!=None:
2281 2289 self.byRanges = True
2282 2290 else:
2283 2291 if minHei==None or maxHei==None :
2284 2292 raise ValueError("Parameters heights are required")
2285 2293 if minRef==None or maxRef==None:
2286 2294 raise ValueError("Parameters heights are required")
2287 2295
2288 2296 if self.byRanges:
2289 2297 self.min_sats = []
2290 2298 self.max_sats = []
2291 2299 for min,max in rangeHeiList:
2292 2300 a,b = getHei_index(min, max, dataOut.heightList)
2293 2301 self.min_sats.append(a)
2294 2302 self.max_sats.append(b)
2295 2303 else:
2296 2304 self.min_sats, self.max_sats = getHei_index(minHei, maxHei, dataOut.heightList)
2297 2305 self.min_ref, self.max_ref = getHei_index(minRef, maxRef, dataOut.heightList)
2298 2306 self.th = th
2299 2307 self.thdB = thdB
2300 2308 self.isConfig = True
2301 2309
2302 2310
2303 2311 def compareRanges(self,data, minHei,maxHei):
2304 2312
2305 2313 # ref = data[0,self.min_ref:self.max_ref] * numpy.conjugate(data[0,self.min_ref:self.max_ref])
2306 2314 # p_ref = 10*numpy.log10(ref.real)
2307 2315 # m_ref = numpy.mean(p_ref)
2308 2316
2309 2317 m_ref = self.noise
2310 2318
2311 2319 sats = data[0,minHei:maxHei] * numpy.conjugate(data[0,minHei:maxHei])
2312 2320 p_sats = 10*numpy.log10(sats.real)
2313 2321 m_sats = numpy.mean(p_sats)
2314 2322
2315 2323 if m_sats > (m_ref + self.th): #and (m_sats > self.thdB):
2316 2324 #print("msats: ",m_sats," \tmRef: ", m_ref, "\t",(m_sats - m_ref))
2317 2325 #print("Removing profiles...")
2318 2326 return False
2319 2327
2320 2328 return True
2321 2329
2322 2330 def isProfileClean(self, data):
2323 2331 '''
2324 2332 Analiza solo 1 canal, y descarta todos...
2325 2333 '''
2326 2334
2327 2335 clean = True
2328 2336
2329 2337 if self.byRanges:
2330 2338
2331 2339 for n in range(len(self.min_sats)):
2332 2340 c = self.compareRanges(data,self.min_sats[n],self.max_sats[n])
2333 2341 clean = clean and c
2334 2342 else:
2335 2343
2336 2344 clean = (self.compareRanges(data, self.min_sats,self.max_sats))
2337 2345
2338 2346 return clean
2339 2347
2340 2348
2341 2349
2342 2350 def run(self, dataOut, minHei=None, maxHei=None, minRef=None, maxRef=None, th=5, thdB=65, rangeHeiList=None):
2343 2351 dataOut.flagNoData = True
2344 2352
2345 2353 if not self.isConfig:
2346 2354 self.setup(dataOut, minHei, maxHei, minRef, maxRef, th, thdB, rangeHeiList)
2347 2355 self.isConfig = True
2348 2356 #print(self.min_sats,self.max_sats)
2349 2357 if dataOut.flagDataAsBlock:
2350 2358 raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
2351 2359
2352 2360 else:
2353 2361 self.noise =10*numpy.log10(dataOut.getNoisebyHildebrand(ymin_index=self.min_ref, ymax_index=self.max_ref))
2354 2362 if not self.isProfileClean(dataOut.data):
2355 2363 return dataOut
2356 2364 #dataOut.data = numpy.full((dataOut.nChannels,dataOut.nHeights),numpy.NAN)
2357 2365 #self.count += 1
2358 2366
2359 2367 dataOut.flagNoData = False
2360 2368
2361 2369 return dataOut
General Comments 0
You need to be logged in to leave comments. Login now