##// END OF EJS Templates
schain
RSS
Eliminación de perfiles, retención de bloques, salida en pequeñps bloques o perfiles, rti de outliers
joabAM -
r1528:9eee9f011a2c
parent child
Show More
Side by Side Unified
Context file:
r1528:9eee9f011a2c Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,126 +1,126
1 #include <Python.h>
1 #include <Python.h>
2 #include <numpy/arrayobject.h>
2 #include <numpy/arrayobject.h>
3 #include <math.h>
3 #include <math.h>
4
4
5
5
6 static PyObject *hildebrand_sekhon(PyObject *self, PyObject *args) {
6 static PyObject *hildebrand_sekhon(PyObject *self, PyObject *args) {
7 double navg;
7 double navg;
8 PyObject *data_obj, *data_array;
8 PyObject *data_obj, *data_array;
9
9
10 if (!PyArg_ParseTuple(args, "Od", &data_obj, &navg)) {
10 if (!PyArg_ParseTuple(args, "Od", &data_obj, &navg)) {
11 return NULL;
11 return NULL;
12 }
12 }
13
13
14 data_array = PyArray_FROM_OTF(data_obj, NPY_FLOAT64, NPY_IN_ARRAY);
14 data_array = PyArray_FROM_OTF(data_obj, NPY_FLOAT64, NPY_IN_ARRAY);
15
15
16 if (data_array == NULL) {
16 if (data_array == NULL) {
17 Py_XDECREF(data_array);
17 Py_XDECREF(data_array);
18 Py_XDECREF(data_obj);
18 Py_XDECREF(data_obj);
19 return NULL;
19 return NULL;
20 }
20 }
21 double *sortdata = (double*)PyArray_DATA(data_array);
21 double *sortdata = (double*)PyArray_DATA(data_array);
22 int lenOfData = (int)PyArray_SIZE(data_array) ;
22 int lenOfData = (int)PyArray_SIZE(data_array) ;
23 double nums_min = lenOfData*0.2;
23 double nums_min = lenOfData*0.2;
24 if (nums_min <= 5) nums_min = 5;
24 if (nums_min <= 5) nums_min = 5;
25 double sump = 0;
25 double sump = 0;
26 double sumq = 0;
26 double sumq = 0;
27 long j = 0;
27 long j = 0;
28 int cont = 1;
28 int cont = 1;
29 double rtest = 0;
29 double rtest = 0;
30 while ((cont == 1) && (j < lenOfData)) {
30 while ((cont == 1) && (j < lenOfData)) {
31 sump = sump + sortdata[j];
31 sump = sump + sortdata[j];
32 sumq = sumq + pow(sortdata[j], 2);
32 sumq = sumq + pow(sortdata[j], 2);
33 if (j > nums_min) {
33 if (j > nums_min) {
34 rtest = (double)j/(j-1) + 1/navg;
34 rtest = (double)j/(j-1) + 1/navg;
35 if ((sumq*j) > (rtest*pow(sump, 2))) {
35 if ((sumq*j) > (rtest*pow(sump, 2))) {
36 j = j - 1;
36 j = j - 1;
37 sump = sump - sortdata[j];
37 sump = sump - sortdata[j];
38 sumq = sumq - pow(sortdata[j],2);
38 sumq = sumq - pow(sortdata[j],2);
39 cont = 0;
39 cont = 0;
40 }
40 }
41 }
41 }
42 j = j + 1;
42 j = j + 1;
43 }
43 }
44
44
45 double lnoise = sump / j;
45 double lnoise = sump / j;
46
46
47 Py_DECREF(data_array);
47 Py_DECREF(data_array);
48
48
49 // return PyLong_FromLong(lnoise);
49 // return PyLong_FromLong(lnoise);
50 return PyFloat_FromDouble(lnoise);
50 return PyFloat_FromDouble(lnoise);
51 }
51 }
52
52
53 static PyObject *hildebrand_sekhon2(PyObject *self, PyObject *args) {
53 static PyObject *hildebrand_sekhon2(PyObject *self, PyObject *args) {
54 double navg;
54 double navg;
55 PyObject *data_obj, *data_array;
55 PyObject *data_obj, *data_array;
56
56
57 if (!PyArg_ParseTuple(args, "Od", &data_obj, &navg)) {
57 if (!PyArg_ParseTuple(args, "Od", &data_obj, &navg)) {
58 return NULL;
58 return NULL;
59 }
59 }
60 data_array = PyArray_FROM_OTF(data_obj, NPY_FLOAT64, NPY_IN_ARRAY);
60 data_array = PyArray_FROM_OTF(data_obj, NPY_FLOAT64, NPY_IN_ARRAY);
61
61
62 if (data_array == NULL) {
62 if (data_array == NULL) {
63 Py_XDECREF(data_array);
63 Py_XDECREF(data_array);
64 Py_XDECREF(data_obj);
64 Py_XDECREF(data_obj);
65 return NULL;
65 return NULL;
66 }
66 }
67 double *sortdata = (double*)PyArray_DATA(data_array);
67 double *sortdata = (double*)PyArray_DATA(data_array);
68 int lenOfData = (int)PyArray_SIZE(data_array) ;
68 int lenOfData = (int)PyArray_SIZE(data_array) ;
69 double nums_min = lenOfData*0.75;
69 double nums_min = lenOfData*0.85;
70 if (nums_min <= 5) nums_min = 5;
70 if (nums_min <= 5) nums_min = 5;
71 double sump = 0;
71 double sump = 0;
72 double sumq = 0;
72 double sumq = 0;
73 long j = 0;
73 long j = 0;
74 int cont = 1;
74 int cont = 1;
75 double rtest = 0;
75 double rtest = 0;
76 while ((cont == 1) && (j < lenOfData)) {
76 while ((cont == 1) && (j < lenOfData)) {
77 sump = sump + sortdata[j];
77 sump = sump + sortdata[j];
78 sumq = sumq + pow(sortdata[j], 2);
78 sumq = sumq + pow(sortdata[j], 2);
79 if (j > nums_min) {
79 if (j > nums_min) {
80 rtest = (double)j/(j-1) + 1/navg;
80 rtest = (double)j/(j-1) + 1/navg;
81 if ((sumq*j) > (rtest*pow(sump, 2))) {
81 if ((sumq*j) > (rtest*pow(sump, 2))) {
82 j = j - 1;
82 j = j - 1;
83 sump = sump - sortdata[j];
83 sump = sump - sortdata[j];
84 sumq = sumq - pow(sortdata[j],2);
84 sumq = sumq - pow(sortdata[j],2);
85 cont = 0;
85 cont = 0;
86 }
86 }
87 }
87 }
88 j = j + 1;
88 j = j + 1;
89 }
89 }
90
90
91 Py_DECREF(data_array);
91 Py_DECREF(data_array);
92
92
93 return PyLong_FromLong(j);
93 return PyLong_FromLong(j);
94 }
94 }
95
95
96
96
97 static PyMethodDef noiseMethods[] = {
97 static PyMethodDef noiseMethods[] = {
98 { "hildebrand_sekhon", hildebrand_sekhon, METH_VARARGS, "Get noise with hildebrand_sekhon algorithm" },
98 { "hildebrand_sekhon", hildebrand_sekhon, METH_VARARGS, "Get noise with hildebrand_sekhon algorithm" },
99 { "hildebrand_sekhon2", hildebrand_sekhon2, METH_VARARGS, "Variation for satellite cleaning" },
99 { "hildebrand_sekhon2", hildebrand_sekhon2, METH_VARARGS, "Variation for satellite cleaning" },
100 { NULL, NULL, 0, NULL }
100 { NULL, NULL, 0, NULL }
101 };
101 };
102
102
103 #if PY_MAJOR_VERSION >= 3
103 #if PY_MAJOR_VERSION >= 3
104
104
105 static struct PyModuleDef noisemodule = {
105 static struct PyModuleDef noisemodule = {
106 PyModuleDef_HEAD_INIT,
106 PyModuleDef_HEAD_INIT,
107 "_noise",
107 "_noise",
108 "Get noise with hildebrand_sekhon algorithm",
108 "Get noise with hildebrand_sekhon algorithm",
109 -1,
109 -1,
110 noiseMethods
110 noiseMethods
111 };
111 };
112
112
113 #endif
113 #endif
114
114
115 #if PY_MAJOR_VERSION >= 3
115 #if PY_MAJOR_VERSION >= 3
116 PyMODINIT_FUNC PyInit__noise(void) {
116 PyMODINIT_FUNC PyInit__noise(void) {
117 Py_Initialize();
117 Py_Initialize();
118 import_array();
118 import_array();
119 return PyModule_Create(&noisemodule);
119 return PyModule_Create(&noisemodule);
120 }
120 }
121 #else
121 #else
122 PyMODINIT_FUNC init_noise() {
122 PyMODINIT_FUNC init_noise() {
123 Py_InitModule("_noise", noiseMethods);
123 Py_InitModule("_noise", noiseMethods);
124 import_array();
124 import_array();
125 }
125 }
126 #endif
126 #endif
Show file before | Show file after | Hide comments
@@ -1,687 +1,685
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """API to create signal chain projects
5 """API to create signal chain projects
6
6
7 The API is provide through class: Project
7 The API is provide through class: Project
8 """
8 """
9
9
10 import re
10 import re
11 import sys
11 import sys
12 import ast
12 import ast
13 import datetime
13 import datetime
14 import traceback
14 import traceback
15 import time
15 import time
16 import multiprocessing
16 import multiprocessing
17 from multiprocessing import Process, Queue
17 from multiprocessing import Process, Queue
18 from threading import Thread
18 from threading import Thread
19 from xml.etree.ElementTree import ElementTree, Element, SubElement
19 from xml.etree.ElementTree import ElementTree, Element, SubElement
20
20
21 from schainpy.admin import Alarm, SchainWarning
21 from schainpy.admin import Alarm, SchainWarning
22 from schainpy.model import *
22 from schainpy.model import *
23 from schainpy.utils import log
23 from schainpy.utils import log
24
24
25 if 'darwin' in sys.platform and sys.version_info[0] == 3 and sys.version_info[1] > 7:
25 if 'darwin' in sys.platform and sys.version_info[0] == 3 and sys.version_info[1] > 7:
26 multiprocessing.set_start_method('fork')
26 multiprocessing.set_start_method('fork')
27
27
28 class ConfBase():
28 class ConfBase():
29
29
30 def __init__(self):
30 def __init__(self):
31
31
32 self.id = '0'
32 self.id = '0'
33 self.name = None
33 self.name = None
34 self.priority = None
34 self.priority = None
35 self.parameters = {}
35 self.parameters = {}
36 self.object = None
36 self.object = None
37 self.operations = []
37 self.operations = []
38
38
39 def getId(self):
39 def getId(self):
40
40
41 return self.id
41 return self.id
42
42
43 def getNewId(self):
43 def getNewId(self):
44
44
45 return int(self.id) * 10 + len(self.operations) + 1
45 return int(self.id) * 10 + len(self.operations) + 1
46
46
47 def updateId(self, new_id):
47 def updateId(self, new_id):
48
48
49 self.id = str(new_id)
49 self.id = str(new_id)
50
50
51 n = 1
51 n = 1
52 for conf in self.operations:
52 for conf in self.operations:
53 conf_id = str(int(new_id) * 10 + n)
53 conf_id = str(int(new_id) * 10 + n)
54 conf.updateId(conf_id)
54 conf.updateId(conf_id)
55 n += 1
55 n += 1
56
56
57 def getKwargs(self):
57 def getKwargs(self):
58
58
59 params = {}
59 params = {}
60
60
61 for key, value in self.parameters.items():
61 for key, value in self.parameters.items():
62 if value not in (None, '', ' '):
62 if value not in (None, '', ' '):
63 params[key] = value
63 params[key] = value
64
64
65 return params
65 return params
66
66
67 def update(self, **kwargs):
67 def update(self, **kwargs):
68
68
69 for key, value in kwargs.items():
69 for key, value in kwargs.items():
70 self.addParameter(name=key, value=value)
70 self.addParameter(name=key, value=value)
71
71
72 def addParameter(self, name, value, format=None):
72 def addParameter(self, name, value, format=None):
73 '''
73 '''
74 '''
74 '''
75
75
76 if isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
76 if isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
77 self.parameters[name] = datetime.date(*[int(x) for x in value.split('/')])
77 self.parameters[name] = datetime.date(*[int(x) for x in value.split('/')])
78 elif isinstance(value, str) and re.search(r'(\d+:\d+:\d+)', value):
78 elif isinstance(value, str) and re.search(r'(\d+:\d+:\d+)', value):
79 self.parameters[name] = datetime.time(*[int(x) for x in value.split(':')])
79 self.parameters[name] = datetime.time(*[int(x) for x in value.split(':')])
80 else:
80 else:
81 try:
81 try:
82 self.parameters[name] = ast.literal_eval(value)
82 self.parameters[name] = ast.literal_eval(value)
83 except:
83 except:
84 if isinstance(value, str) and ',' in value:
84 if isinstance(value, str) and ',' in value:
85 self.parameters[name] = value.split(',')
85 self.parameters[name] = value.split(',')
86 else:
86 else:
87 self.parameters[name] = value
87 self.parameters[name] = value
88
88
89 def getParameters(self):
89 def getParameters(self):
90
90
91 params = {}
91 params = {}
92 for key, value in self.parameters.items():
92 for key, value in self.parameters.items():
93 s = type(value).__name__
93 s = type(value).__name__
94 if s == 'date':
94 if s == 'date':
95 params[key] = value.strftime('%Y/%m/%d')
95 params[key] = value.strftime('%Y/%m/%d')
96 elif s == 'time':
96 elif s == 'time':
97 params[key] = value.strftime('%H:%M:%S')
97 params[key] = value.strftime('%H:%M:%S')
98 else:
98 else:
99 params[key] = str(value)
99 params[key] = str(value)
100
100
101 return params
101 return params
102
102
103 def makeXml(self, element):
103 def makeXml(self, element):
104
104
105 xml = SubElement(element, self.ELEMENTNAME)
105 xml = SubElement(element, self.ELEMENTNAME)
106 for label in self.xml_labels:
106 for label in self.xml_labels:
107 xml.set(label, str(getattr(self, label)))
107 xml.set(label, str(getattr(self, label)))
108
108
109 for key, value in self.getParameters().items():
109 for key, value in self.getParameters().items():
110 xml_param = SubElement(xml, 'Parameter')
110 xml_param = SubElement(xml, 'Parameter')
111 xml_param.set('name', key)
111 xml_param.set('name', key)
112 xml_param.set('value', value)
112 xml_param.set('value', value)
113
113
114 for conf in self.operations:
114 for conf in self.operations:
115 conf.makeXml(xml)
115 conf.makeXml(xml)
116
116
117 def __str__(self):
117 def __str__(self):
118
118
119 if self.ELEMENTNAME == 'Operation':
119 if self.ELEMENTNAME == 'Operation':
120 s = ' {}[id={}]\n'.format(self.name, self.id)
120 s = ' {}[id={}]\n'.format(self.name, self.id)
121 else:
121 else:
122 s = '{}[id={}, inputId={}]\n'.format(self.name, self.id, self.inputId)
122 s = '{}[id={}, inputId={}]\n'.format(self.name, self.id, self.inputId)
123
123
124 for key, value in self.parameters.items():
124 for key, value in self.parameters.items():
125 if self.ELEMENTNAME == 'Operation':
125 if self.ELEMENTNAME == 'Operation':
126 s += ' {}: {}\n'.format(key, value)
126 s += ' {}: {}\n'.format(key, value)
127 else:
127 else:
128 s += ' {}: {}\n'.format(key, value)
128 s += ' {}: {}\n'.format(key, value)
129
129
130 for conf in self.operations:
130 for conf in self.operations:
131 s += str(conf)
131 s += str(conf)
132
132
133 return s
133 return s
134
134
135 class OperationConf(ConfBase):
135 class OperationConf(ConfBase):
136
136
137 ELEMENTNAME = 'Operation'
137 ELEMENTNAME = 'Operation'
138 xml_labels = ['id', 'name']
138 xml_labels = ['id', 'name']
139
139
140 def setup(self, id, name, priority, project_id, err_queue):
140 def setup(self, id, name, priority, project_id, err_queue):
141
141
142 self.id = str(id)
142 self.id = str(id)
143 self.project_id = project_id
143 self.project_id = project_id
144 self.name = name
144 self.name = name
145 self.type = 'other'
145 self.type = 'other'
146 self.err_queue = err_queue
146 self.err_queue = err_queue
147
147
148 def readXml(self, element, project_id, err_queue):
148 def readXml(self, element, project_id, err_queue):
149
149
150 self.id = element.get('id')
150 self.id = element.get('id')
151 self.name = element.get('name')
151 self.name = element.get('name')
152 self.type = 'other'
152 self.type = 'other'
153 self.project_id = str(project_id)
153 self.project_id = str(project_id)
154 self.err_queue = err_queue
154 self.err_queue = err_queue
155
155
156 for elm in element.iter('Parameter'):
156 for elm in element.iter('Parameter'):
157 self.addParameter(elm.get('name'), elm.get('value'))
157 self.addParameter(elm.get('name'), elm.get('value'))
158
158
159 def createObject(self):
159 def createObject(self):
160
160
161 className = eval(self.name)
161 className = eval(self.name)
162
162
163 if 'Plot' in self.name or 'Writer' in self.name or 'Send' in self.name or 'print' in self.name:
163 if 'Plot' in self.name or 'Writer' in self.name or 'Send' in self.name or 'print' in self.name:
164 kwargs = self.getKwargs()
164 kwargs = self.getKwargs()
165 opObj = className(self.name, self.id, self.project_id, self.err_queue, **kwargs)
165 opObj = className(self.name, self.id, self.project_id, self.err_queue, **kwargs)
166 opObj.start()
166 opObj.start()
167 self.type = 'external'
167 self.type = 'external'
168 else:
168 else:
169 opObj = className()
169 opObj = className()
170
170
171 self.object = opObj
171 self.object = opObj
172 return opObj
172 return opObj
173
173
174 class ProcUnitConf(ConfBase):
174 class ProcUnitConf(ConfBase):
175
175
176 ELEMENTNAME = 'ProcUnit'
176 ELEMENTNAME = 'ProcUnit'
177 xml_labels = ['id', 'inputId', 'name']
177 xml_labels = ['id', 'inputId', 'name']
178
178
179 def setup(self, project_id, id, name, datatype, inputId, err_queue):
179 def setup(self, project_id, id, name, datatype, inputId, err_queue):
180 '''
180 '''
181 '''
181 '''
182
182
183 if datatype == None and name == None:
183 if datatype == None and name == None:
184 raise ValueError('datatype or name should be defined')
184 raise ValueError('datatype or name should be defined')
185
185
186 if name == None:
186 if name == None:
187 if 'Proc' in datatype:
187 if 'Proc' in datatype:
188 name = datatype
188 name = datatype
189 else:
189 else:
190 name = '%sProc' % (datatype)
190 name = '%sProc' % (datatype)
191
191
192 if datatype == None:
192 if datatype == None:
193 datatype = name.replace('Proc', '')
193 datatype = name.replace('Proc', '')
194
194
195 self.id = str(id)
195 self.id = str(id)
196 self.project_id = project_id
196 self.project_id = project_id
197 self.name = name
197 self.name = name
198 self.datatype = datatype
198 self.datatype = datatype
199 self.inputId = inputId
199 self.inputId = inputId
200 self.err_queue = err_queue
200 self.err_queue = err_queue
201 self.operations = []
201 self.operations = []
202 self.parameters = {}
202 self.parameters = {}
203
203
204 def removeOperation(self, id):
204 def removeOperation(self, id):
205
205
206 i = [1 if x.id==id else 0 for x in self.operations]
206 i = [1 if x.id==id else 0 for x in self.operations]
207 self.operations.pop(i.index(1))
207 self.operations.pop(i.index(1))
208
208
209 def getOperation(self, id):
209 def getOperation(self, id):
210
210
211 for conf in self.operations:
211 for conf in self.operations:
212 if conf.id == id:
212 if conf.id == id:
213 return conf
213 return conf
214
214
215 def addOperation(self, name, optype='self'):
215 def addOperation(self, name, optype='self'):
216 '''
216 '''
217 '''
217 '''
218
218
219 id = self.getNewId()
219 id = self.getNewId()
220 conf = OperationConf()
220 conf = OperationConf()
221 conf.setup(id, name=name, priority='0', project_id=self.project_id, err_queue=self.err_queue)
221 conf.setup(id, name=name, priority='0', project_id=self.project_id, err_queue=self.err_queue)
222 self.operations.append(conf)
222 self.operations.append(conf)
223
223
224 return conf
224 return conf
225
225
226 def readXml(self, element, project_id, err_queue):
226 def readXml(self, element, project_id, err_queue):
227
227
228 self.id = element.get('id')
228 self.id = element.get('id')
229 self.name = element.get('name')
229 self.name = element.get('name')
230 self.inputId = None if element.get('inputId') == 'None' else element.get('inputId')
230 self.inputId = None if element.get('inputId') == 'None' else element.get('inputId')
231 self.datatype = element.get('datatype', self.name.replace(self.ELEMENTNAME.replace('Unit', ''), ''))
231 self.datatype = element.get('datatype', self.name.replace(self.ELEMENTNAME.replace('Unit', ''), ''))
232 self.project_id = str(project_id)
232 self.project_id = str(project_id)
233 self.err_queue = err_queue
233 self.err_queue = err_queue
234 self.operations = []
234 self.operations = []
235 self.parameters = {}
235 self.parameters = {}
236
236
237 for elm in element:
237 for elm in element:
238 if elm.tag == 'Parameter':
238 if elm.tag == 'Parameter':
239 self.addParameter(elm.get('name'), elm.get('value'))
239 self.addParameter(elm.get('name'), elm.get('value'))
240 elif elm.tag == 'Operation':
240 elif elm.tag == 'Operation':
241 conf = OperationConf()
241 conf = OperationConf()
242 conf.readXml(elm, project_id, err_queue)
242 conf.readXml(elm, project_id, err_queue)
243 self.operations.append(conf)
243 self.operations.append(conf)
244
244
245 def createObjects(self):
245 def createObjects(self):
246 '''
246 '''
247 Instancia de unidades de procesamiento.
247 Instancia de unidades de procesamiento.
248 '''
248 '''
249
249
250 className = eval(self.name)
250 className = eval(self.name)
251 kwargs = self.getKwargs()
251 kwargs = self.getKwargs()
252 procUnitObj = className()
252 procUnitObj = className()
253 procUnitObj.name = self.name
253 procUnitObj.name = self.name
254 log.success('creating process... id: {}, inputId: {}'.format(self.id,self.inputId), self.name)
254 log.success('creating process... id: {}, inputId: {}'.format(self.id,self.inputId), self.name)
255
255
256 for conf in self.operations:
256 for conf in self.operations:
257
257
258 opObj = conf.createObject()
258 opObj = conf.createObject()
259
259
260 log.success('adding operation: {}, type:{}'.format(conf.name,conf.type), self.name)
260 log.success('adding operation: {}, type:{}'.format(conf.name,conf.type), self.name)
261
261
262 procUnitObj.addOperation(conf, opObj)
262 procUnitObj.addOperation(conf, opObj)
263
263
264 self.object = procUnitObj
264 self.object = procUnitObj
265
265
266 def run(self):
266 def run(self):
267 '''
267 '''
268 '''
268 '''
269 return self.object.call(**self.getKwargs())
269 return self.object.call(**self.getKwargs())
270
270
271 class ReadUnitConf(ProcUnitConf):
271 class ReadUnitConf(ProcUnitConf):
272
272
273 ELEMENTNAME = 'ReadUnit'
273 ELEMENTNAME = 'ReadUnit'
274
274
275 def __init__(self):
275 def __init__(self):
276
276
277 self.id = None
277 self.id = None
278 self.datatype = None
278 self.datatype = None
279 self.name = None
279 self.name = None
280 self.inputId = None
280 self.inputId = None
281 self.operations = []
281 self.operations = []
282 self.parameters = {}
282 self.parameters = {}
283
283
284 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
284 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
285 startTime='', endTime='', server=None, **kwargs):
285 startTime='', endTime='', server=None, **kwargs):
286
286
287 if datatype == None and name == None:
287 if datatype == None and name == None:
288 raise ValueError('datatype or name should be defined')
288 raise ValueError('datatype or name should be defined')
289 if name == None:
289 if name == None:
290 if 'Reader' in datatype:
290 if 'Reader' in datatype:
291 name = datatype
291 name = datatype
292 datatype = name.replace('Reader','')
292 datatype = name.replace('Reader','')
293 else:
293 else:
294 name = '{}Reader'.format(datatype)
294 name = '{}Reader'.format(datatype)
295 if datatype == None:
295 if datatype == None:
296 if 'Reader' in name:
296 if 'Reader' in name:
297 datatype = name.replace('Reader','')
297 datatype = name.replace('Reader','')
298 else:
298 else:
299 datatype = name
299 datatype = name
300 name = '{}Reader'.format(name)
300 name = '{}Reader'.format(name)
301
301
302 self.id = id
302 self.id = id
303 self.project_id = project_id
303 self.project_id = project_id
304 self.name = name
304 self.name = name
305 self.datatype = datatype
305 self.datatype = datatype
306 self.err_queue = err_queue
306 self.err_queue = err_queue
307
307
308 self.addParameter(name='path', value=path)
308 self.addParameter(name='path', value=path)
309 self.addParameter(name='startDate', value=startDate)
309 self.addParameter(name='startDate', value=startDate)
310 self.addParameter(name='endDate', value=endDate)
310 self.addParameter(name='endDate', value=endDate)
311 self.addParameter(name='startTime', value=startTime)
311 self.addParameter(name='startTime', value=startTime)
312 self.addParameter(name='endTime', value=endTime)
312 self.addParameter(name='endTime', value=endTime)
313
313
314 for key, value in kwargs.items():
314 for key, value in kwargs.items():
315 self.addParameter(name=key, value=value)
315 self.addParameter(name=key, value=value)
316
316
317
317
318 class Project(Process):
318 class Project(Process):
319 """API to create signal chain projects"""
319 """API to create signal chain projects"""
320
320
321 ELEMENTNAME = 'Project'
321 ELEMENTNAME = 'Project'
322
322
323 def __init__(self, name=''):
323 def __init__(self, name=''):
324
324
325 Process.__init__(self)
325 Process.__init__(self)
326 self.id = '1'
326 self.id = '1'
327 if name:
327 if name:
328 self.name = '{} ({})'.format(Process.__name__, name)
328 self.name = '{} ({})'.format(Process.__name__, name)
329 self.filename = None
329 self.filename = None
330 self.description = None
330 self.description = None
331 self.email = None
331 self.email = None
332 self.alarm = []
332 self.alarm = []
333 self.configurations = {}
333 self.configurations = {}
334 # self.err_queue = Queue()
334 # self.err_queue = Queue()
335 self.err_queue = None
335 self.err_queue = None
336 self.started = False
336 self.started = False
337
337
338 def getNewId(self):
338 def getNewId(self):
339
339
340 idList = list(self.configurations.keys())
340 idList = list(self.configurations.keys())
341 id = int(self.id) * 10
341 id = int(self.id) * 10
342
342
343 while True:
343 while True:
344 id += 1
344 id += 1
345
345
346 if str(id) in idList:
346 if str(id) in idList:
347 continue
347 continue
348
348
349 break
349 break
350
350
351 return str(id)
351 return str(id)
352
352
353 def updateId(self, new_id):
353 def updateId(self, new_id):
354
354
355 self.id = str(new_id)
355 self.id = str(new_id)
356
356
357 keyList = list(self.configurations.keys())
357 keyList = list(self.configurations.keys())
358 keyList.sort()
358 keyList.sort()
359
359
360 n = 1
360 n = 1
361 new_confs = {}
361 new_confs = {}
362
362
363 for procKey in keyList:
363 for procKey in keyList:
364
364
365 conf = self.configurations[procKey]
365 conf = self.configurations[procKey]
366 idProcUnit = str(int(self.id) * 10 + n)
366 idProcUnit = str(int(self.id) * 10 + n)
367 conf.updateId(idProcUnit)
367 conf.updateId(idProcUnit)
368 new_confs[idProcUnit] = conf
368 new_confs[idProcUnit] = conf
369 n += 1
369 n += 1
370
370
371 self.configurations = new_confs
371 self.configurations = new_confs
372
372
373 def setup(self, id=1, name='', description='', email=None, alarm=[]):
373 def setup(self, id=1, name='', description='', email=None, alarm=[]):
374
374
375 self.id = str(id)
375 self.id = str(id)
376 self.description = description
376 self.description = description
377 self.email = email
377 self.email = email
378 self.alarm = alarm
378 self.alarm = alarm
379 if name:
379 if name:
380 self.name = '{} ({})'.format(Process.__name__, name)
380 self.name = '{} ({})'.format(Process.__name__, name)
381
381
382 def update(self, **kwargs):
382 def update(self, **kwargs):
383
383
384 for key, value in kwargs.items():
384 for key, value in kwargs.items():
385 setattr(self, key, value)
385 setattr(self, key, value)
386
386
387 def clone(self):
387 def clone(self):
388
388
389 p = Project()
389 p = Project()
390 p.id = self.id
390 p.id = self.id
391 p.name = self.name
391 p.name = self.name
392 p.description = self.description
392 p.description = self.description
393 p.configurations = self.configurations.copy()
393 p.configurations = self.configurations.copy()
394
394
395 return p
395 return p
396
396
397 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
397 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
398
398
399 '''
399 '''
400 '''
400 '''
401
401
402 if id is None:
402 if id is None:
403 idReadUnit = self.getNewId()
403 idReadUnit = self.getNewId()
404 else:
404 else:
405 idReadUnit = str(id)
405 idReadUnit = str(id)
406
406
407 conf = ReadUnitConf()
407 conf = ReadUnitConf()
408 conf.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
408 conf.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
409 self.configurations[conf.id] = conf
409 self.configurations[conf.id] = conf
410
410
411 return conf
411 return conf
412
412
413 def addProcUnit(self, id=None, inputId='0', datatype=None, name=None):
413 def addProcUnit(self, id=None, inputId='0', datatype=None, name=None):
414
414
415 '''
415 '''
416 '''
416 '''
417
417
418 if id is None:
418 if id is None:
419 idProcUnit = self.getNewId()
419 idProcUnit = self.getNewId()
420 else:
420 else:
421 idProcUnit = id
421 idProcUnit = id
422
422
423 conf = ProcUnitConf()
423 conf = ProcUnitConf()
424 conf.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
424 conf.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
425 self.configurations[conf.id] = conf
425 self.configurations[conf.id] = conf
426
426
427 return conf
427 return conf
428
428
429 def removeProcUnit(self, id):
429 def removeProcUnit(self, id):
430
430
431 if id in self.configurations:
431 if id in self.configurations:
432 self.configurations.pop(id)
432 self.configurations.pop(id)
433
433
434 def getReadUnit(self):
434 def getReadUnit(self):
435
435
436 for obj in list(self.configurations.values()):
436 for obj in list(self.configurations.values()):
437 if obj.ELEMENTNAME == 'ReadUnit':
437 if obj.ELEMENTNAME == 'ReadUnit':
438 return obj
438 return obj
439
439
440 return None
440 return None
441
441
442 def getProcUnit(self, id):
442 def getProcUnit(self, id):
443
443
444 return self.configurations[id]
444 return self.configurations[id]
445
445
446 def getUnits(self):
446 def getUnits(self):
447
447
448 keys = list(self.configurations)
448 keys = list(self.configurations)
449 keys.sort()
449 keys.sort()
450
450
451 for key in keys:
451 for key in keys:
452 yield self.configurations[key]
452 yield self.configurations[key]
453
453
454 def updateUnit(self, id, **kwargs):
454 def updateUnit(self, id, **kwargs):
455
455
456 conf = self.configurations[id].update(**kwargs)
456 conf = self.configurations[id].update(**kwargs)
457
457
458 def makeXml(self):
458 def makeXml(self):
459
459
460 xml = Element('Project')
460 xml = Element('Project')
461 xml.set('id', str(self.id))
461 xml.set('id', str(self.id))
462 xml.set('name', self.name)
462 xml.set('name', self.name)
463 xml.set('description', self.description)
463 xml.set('description', self.description)
464
464
465 for conf in self.configurations.values():
465 for conf in self.configurations.values():
466 conf.makeXml(xml)
466 conf.makeXml(xml)
467
467
468 self.xml = xml
468 self.xml = xml
469
469
470 def writeXml(self, filename=None):
470 def writeXml(self, filename=None):
471
471
472 if filename == None:
472 if filename == None:
473 if self.filename:
473 if self.filename:
474 filename = self.filename
474 filename = self.filename
475 else:
475 else:
476 filename = 'schain.xml'
476 filename = 'schain.xml'
477
477
478 if not filename:
478 if not filename:
479 print('filename has not been defined. Use setFilename(filename) for do it.')
479 print('filename has not been defined. Use setFilename(filename) for do it.')
480 return 0
480 return 0
481
481
482 abs_file = os.path.abspath(filename)
482 abs_file = os.path.abspath(filename)
483
483
484 if not os.access(os.path.dirname(abs_file), os.W_OK):
484 if not os.access(os.path.dirname(abs_file), os.W_OK):
485 print('No write permission on %s' % os.path.dirname(abs_file))
485 print('No write permission on %s' % os.path.dirname(abs_file))
486 return 0
486 return 0
487
487
488 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
488 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
489 print('File %s already exists and it could not be overwriten' % abs_file)
489 print('File %s already exists and it could not be overwriten' % abs_file)
490 return 0
490 return 0
491
491
492 self.makeXml()
492 self.makeXml()
493
493
494 ElementTree(self.xml).write(abs_file, method='xml')
494 ElementTree(self.xml).write(abs_file, method='xml')
495
495
496 self.filename = abs_file
496 self.filename = abs_file
497
497
498 return 1
498 return 1
499
499
500 def readXml(self, filename):
500 def readXml(self, filename):
501
501
502 abs_file = os.path.abspath(filename)
502 abs_file = os.path.abspath(filename)
503
503
504 self.configurations = {}
504 self.configurations = {}
505
505
506 try:
506 try:
507 self.xml = ElementTree().parse(abs_file)
507 self.xml = ElementTree().parse(abs_file)
508 except:
508 except:
509 log.error('Error reading %s, verify file format' % filename)
509 log.error('Error reading %s, verify file format' % filename)
510 return 0
510 return 0
511
511
512 self.id = self.xml.get('id')
512 self.id = self.xml.get('id')
513 self.name = self.xml.get('name')
513 self.name = self.xml.get('name')
514 self.description = self.xml.get('description')
514 self.description = self.xml.get('description')
515
515
516 for element in self.xml:
516 for element in self.xml:
517 if element.tag == 'ReadUnit':
517 if element.tag == 'ReadUnit':
518 conf = ReadUnitConf()
518 conf = ReadUnitConf()
519 conf.readXml(element, self.id, self.err_queue)
519 conf.readXml(element, self.id, self.err_queue)
520 self.configurations[conf.id] = conf
520 self.configurations[conf.id] = conf
521 elif element.tag == 'ProcUnit':
521 elif element.tag == 'ProcUnit':
522 conf = ProcUnitConf()
522 conf = ProcUnitConf()
523 input_proc = self.configurations[element.get('inputId')]
523 input_proc = self.configurations[element.get('inputId')]
524 conf.readXml(element, self.id, self.err_queue)
524 conf.readXml(element, self.id, self.err_queue)
525 self.configurations[conf.id] = conf
525 self.configurations[conf.id] = conf
526
526
527 self.filename = abs_file
527 self.filename = abs_file
528
528
529 return 1
529 return 1
530
530
531 def __str__(self):
531 def __str__(self):
532
532
533 text = '\nProject[id=%s, name=%s, description=%s]\n\n' % (
533 text = '\nProject[id=%s, name=%s, description=%s]\n\n' % (
534 self.id,
534 self.id,
535 self.name,
535 self.name,
536 self.description,
536 self.description,
537 )
537 )
538
538
539 for conf in self.configurations.values():
539 for conf in self.configurations.values():
540 text += '{}'.format(conf)
540 text += '{}'.format(conf)
541
541
542 return text
542 return text
543
543
544 def createObjects(self):
544 def createObjects(self):
545
545
546 keys = list(self.configurations.keys())
546 keys = list(self.configurations.keys())
547 keys.sort()
547 keys.sort()
548 for key in keys:
548 for key in keys:
549 conf = self.configurations[key]
549 conf = self.configurations[key]
550 conf.createObjects()
550 conf.createObjects()
551 if conf.inputId is not None:
551 if conf.inputId is not None:
552 conf.object.setInput(self.configurations[conf.inputId].object)
552 conf.object.setInput(self.configurations[conf.inputId].object)
553
553
554 def monitor(self):
554 def monitor(self):
555
555
556 t = Thread(target=self._monitor, args=(self.err_queue, self.ctx))
556 t = Thread(target=self._monitor, args=(self.err_queue, self.ctx))
557 t.start()
557 t.start()
558
558
559 def _monitor(self, queue, ctx):
559 def _monitor(self, queue, ctx):
560
560
561 import socket
561 import socket
562
562
563 procs = 0
563 procs = 0
564 err_msg = ''
564 err_msg = ''
565
565
566 while True:
566 while True:
567 msg = queue.get()
567 msg = queue.get()
568 if '#_start_#' in msg:
568 if '#_start_#' in msg:
569 procs += 1
569 procs += 1
570 elif '#_end_#' in msg:
570 elif '#_end_#' in msg:
571 procs -=1
571 procs -=1
572 else:
572 else:
573 err_msg = msg
573 err_msg = msg
574
574
575 if procs == 0 or 'Traceback' in err_msg:
575 if procs == 0 or 'Traceback' in err_msg:
576 break
576 break
577 time.sleep(0.1)
577 time.sleep(0.1)
578
578
579 if '|' in err_msg:
579 if '|' in err_msg:
580 name, err = err_msg.split('|')
580 name, err = err_msg.split('|')
581 if 'SchainWarning' in err:
581 if 'SchainWarning' in err:
582 log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
582 log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
583 elif 'SchainError' in err:
583 elif 'SchainError' in err:
584 log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
584 log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
585 else:
585 else:
586 log.error(err, name)
586 log.error(err, name)
587 else:
587 else:
588 name, err = self.name, err_msg
588 name, err = self.name, err_msg
589
589
590 time.sleep(1)
590 time.sleep(1)
591
591
592 ctx.term()
592 ctx.term()
593
593
594 message = ''.join(err)
594 message = ''.join(err)
595
595
596 if err_msg:
596 if err_msg:
597 subject = 'SChain v%s: Error running %s\n' % (
597 subject = 'SChain v%s: Error running %s\n' % (
598 schainpy.__version__, self.name)
598 schainpy.__version__, self.name)
599
599
600 subtitle = 'Hostname: %s\n' % socket.gethostbyname(
600 subtitle = 'Hostname: %s\n' % socket.gethostbyname(
601 socket.gethostname())
601 socket.gethostname())
602 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
602 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
603 subtitle += 'Configuration file: %s\n' % self.filename
603 subtitle += 'Configuration file: %s\n' % self.filename
604 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
604 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
605
605
606 readUnitConfObj = self.getReadUnit()
606 readUnitConfObj = self.getReadUnit()
607 if readUnitConfObj:
607 if readUnitConfObj:
608 subtitle += '\nInput parameters:\n'
608 subtitle += '\nInput parameters:\n'
609 subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters['path']
609 subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters['path']
610 subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters['startDate']
610 subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters['startDate']
611 subtitle += '[End date = %s]\n' % readUnitConfObj.parameters['endDate']
611 subtitle += '[End date = %s]\n' % readUnitConfObj.parameters['endDate']
612 subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters['startTime']
612 subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters['startTime']
613 subtitle += '[End time = %s]\n' % readUnitConfObj.parameters['endTime']
613 subtitle += '[End time = %s]\n' % readUnitConfObj.parameters['endTime']
614
614
615 a = Alarm(
615 a = Alarm(
616 modes=self.alarm,
616 modes=self.alarm,
617 email=self.email,
617 email=self.email,
618 message=message,
618 message=message,
619 subject=subject,
619 subject=subject,
620 subtitle=subtitle,
620 subtitle=subtitle,
621 filename=self.filename
621 filename=self.filename
622 )
622 )
623
623
624 a.start()
624 a.start()
625
625
626 def setFilename(self, filename):
626 def setFilename(self, filename):
627
627
628 self.filename = filename
628 self.filename = filename
629
629
630 def runProcs(self):
630 def runProcs(self):
631
631
632 err = False
632 err = False
633 n = len(self.configurations)
633 n = len(self.configurations)
634 flag_no_read = False
634 flag_no_read = False
635 nProc_noRead = 0
635 nProc_noRead = 0
636 while not err:
636 while not err:
637 #print("STAR")
637 #print("STAR")
638 n_proc = 0
638 n_proc = 0
639 for conf in self.getUnits():
639 for conf in self.getUnits():
640 #print("CONF: ",conf)
640 #print("CONF: ",conf)
641 #print( n_proc, nProc_noRead,flag_no_read)
641 #print( n_proc, nProc_noRead,flag_no_read)
642 if flag_no_read:
642 if flag_no_read:
643 if n_proc >= nProc_noRead:
643 if n_proc >= nProc_noRead:
644 ok = conf.run()
644 ok = conf.run()
645 else:
645 else:
646 #print("break")
646 #print("break")
647 #print(ok, n_proc, nProc_noRead, n, flag_no_read)
647 #print(ok, n_proc, nProc_noRead, n, flag_no_read)
648 n_proc += 1
648 n_proc += 1
649 continue
649 continue
650 else:
650 else:
651 ok = conf.run()
651 ok = conf.run()
652
652
653 n_proc += 1
653 n_proc += 1
654 #print(ok, n_proc, nProc_noRead, n, flag_no_read)
654 #print(ok, n_proc, nProc_noRead, n, flag_no_read)
655
655
656 if ok == 'Error':
656 if ok == 'Error':
657 #self.removeProcUnit(conf.id) #remove proc Unit
657 #self.removeProcUnit(conf.id) #remove proc Unit
658 n -= 1
658 n -= 1
659 continue
659 continue
660
660
661 elif ok == 'no_Read' and (not flag_no_read):
661 elif ok == 'no_Read' and (not flag_no_read):
662 nProc_noRead = n_proc - 1
662 nProc_noRead = n_proc - 1
663 flag_no_read = True
663 flag_no_read = True
664 print("No read")
665 continue
664 continue
666 elif ok == 'new_Read':
665 elif ok == 'new_Read':
667 nProc_noRead = 0
666 nProc_noRead = 0
668 flag_no_read = False
667 flag_no_read = False
669 print("read again")
670 continue
668 continue
671 elif not ok:
669 elif not ok:
672 #print("not ok",ok)
670 #print("not ok",ok)
673 break
671 break
674
672
675 if n == 0:
673 if n == 0:
676 err = True
674 err = True
677
675
678 def run(self):
676 def run(self):
679
677
680 log.success('\nStarting Project {} [id={}]'.format(self.name, self.id), tag='')
678 log.success('\nStarting Project {} [id={}]'.format(self.name, self.id), tag='')
681 self.started = True
679 self.started = True
682 self.start_time = time.time()
680 self.start_time = time.time()
683 self.createObjects()
681 self.createObjects()
684 self.runProcs()
682 self.runProcs()
685 log.success('{} Done (Time: {:4.2f}s)'.format(
683 log.success('{} Done (Time: {:4.2f}s)'.format(
686 self.name,
684 self.name,
687 time.time()-self.start_time), '')
685 time.time()-self.start_time), '')
Show file before | Show file after | Hide comments
@@ -1,1079 +1,1102
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """Definition of diferent Data objects for different types of data
5 """Definition of diferent Data objects for different types of data
6
6
7 Here you will find the diferent data objects for the different types
7 Here you will find the diferent data objects for the different types
8 of data, this data objects must be used as dataIn or dataOut objects in
8 of data, this data objects must be used as dataIn or dataOut objects in
9 processing units and operations. Currently the supported data objects are:
9 processing units and operations. Currently the supported data objects are:
10 Voltage, Spectra, SpectraHeis, Fits, Correlation and Parameters
10 Voltage, Spectra, SpectraHeis, Fits, Correlation and Parameters
11 """
11 """
12
12
13 import copy
13 import copy
14 import numpy
14 import numpy
15 import datetime
15 import datetime
16 import json
16 import json
17
17
18 import schainpy.admin
18 import schainpy.admin
19 from schainpy.utils import log
19 from schainpy.utils import log
20 from .jroheaderIO import SystemHeader, RadarControllerHeader
20 from .jroheaderIO import SystemHeader, RadarControllerHeader
21 from schainpy.model.data import _noise
21 from schainpy.model.data import _noise
22
22
23
23
24 def getNumpyDtype(dataTypeCode):
24 def getNumpyDtype(dataTypeCode):
25
25
26 if dataTypeCode == 0:
26 if dataTypeCode == 0:
27 numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
27 numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
28 elif dataTypeCode == 1:
28 elif dataTypeCode == 1:
29 numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
29 numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
30 elif dataTypeCode == 2:
30 elif dataTypeCode == 2:
31 numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
31 numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
32 elif dataTypeCode == 3:
32 elif dataTypeCode == 3:
33 numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
33 numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
34 elif dataTypeCode == 4:
34 elif dataTypeCode == 4:
35 numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
35 numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
36 elif dataTypeCode == 5:
36 elif dataTypeCode == 5:
37 numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
37 numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
38 else:
38 else:
39 raise ValueError('dataTypeCode was not defined')
39 raise ValueError('dataTypeCode was not defined')
40
40
41 return numpyDtype
41 return numpyDtype
42
42
43
43
44 def getDataTypeCode(numpyDtype):
44 def getDataTypeCode(numpyDtype):
45
45
46 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
46 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
47 datatype = 0
47 datatype = 0
48 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
48 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
49 datatype = 1
49 datatype = 1
50 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
50 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
51 datatype = 2
51 datatype = 2
52 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
52 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
53 datatype = 3
53 datatype = 3
54 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
54 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
55 datatype = 4
55 datatype = 4
56 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
56 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
57 datatype = 5
57 datatype = 5
58 else:
58 else:
59 datatype = None
59 datatype = None
60
60
61 return datatype
61 return datatype
62
62
63
63
64 def hildebrand_sekhon(data, navg):
64 def hildebrand_sekhon(data, navg):
65 """
65 """
66 This method is for the objective determination of the noise level in Doppler spectra. This
66 This method is for the objective determination of the noise level in Doppler spectra. This
67 implementation technique is based on the fact that the standard deviation of the spectral
67 implementation technique is based on the fact that the standard deviation of the spectral
68 densities is equal to the mean spectral density for white Gaussian noise
68 densities is equal to the mean spectral density for white Gaussian noise
69
69
70 Inputs:
70 Inputs:
71 Data : heights
71 Data : heights
72 navg : numbers of averages
72 navg : numbers of averages
73
73
74 Return:
74 Return:
75 mean : noise's level
75 mean : noise's level
76 """
76 """
77
77
78 sortdata = numpy.sort(data, axis=None)
78 sortdata = numpy.sort(data, axis=None)
79 '''
79 '''
80 lenOfData = len(sortdata)
80 lenOfData = len(sortdata)
81 nums_min = lenOfData*0.2
81 nums_min = lenOfData*0.5
82
82
83 if nums_min <= 5:
83 if nums_min <= 5:
84
84
85 nums_min = 5
85 nums_min = 5
86
86
87 sump = 0.
87 sump = 0.
88 sumq = 0.
88 sumq = 0.
89
89
90 j = 0
90 j = 0
91 cont = 1
91 cont = 1
92
92
93 while((cont == 1)and(j < lenOfData)):
93 while((cont == 1)and(j < lenOfData)):
94
94
95 sump += sortdata[j]
95 sump += sortdata[j]
96 sumq += sortdata[j]**2
96 sumq += sortdata[j]**2
97
97
98 if j > nums_min:
98 if j > nums_min:
99 rtest = float(j)/(j-1) + 1.0/navg
99 rtest = float(j)/(j-1) + 1.0/navg
100 if ((sumq*j) > (rtest*sump**2)):
100 if ((sumq*j) > (rtest*sump**2)):
101 j = j - 1
101 j = j - 1
102 sump = sump - sortdata[j]
102 sump = sump - sortdata[j]
103 sumq = sumq - sortdata[j]**2
103 sumq = sumq - sortdata[j]**2
104 cont = 0
104 cont = 0
105
105
106 j += 1
106 j += 1
107
107
108 lnoise = sump / j
108 lnoise = sump / j
109 return lnoise
109 '''
110 '''
110 return _noise.hildebrand_sekhon(sortdata, navg)
111 return _noise.hildebrand_sekhon(sortdata, navg)
111
112
112
113
113 class Beam:
114 class Beam:
114
115
115 def __init__(self):
116 def __init__(self):
116 self.codeList = []
117 self.codeList = []
117 self.azimuthList = []
118 self.azimuthList = []
118 self.zenithList = []
119 self.zenithList = []
119
120
120
121
121
122
122 class GenericData(object):
123 class GenericData(object):
123
124
124 flagNoData = True
125 flagNoData = True
125
126
126 def copy(self, inputObj=None):
127 def copy(self, inputObj=None):
127
128
128 if inputObj == None:
129 if inputObj == None:
129 return copy.deepcopy(self)
130 return copy.deepcopy(self)
130
131
131 for key in list(inputObj.__dict__.keys()):
132 for key in list(inputObj.__dict__.keys()):
132
133
133 attribute = inputObj.__dict__[key]
134 attribute = inputObj.__dict__[key]
134
135
135 # If this attribute is a tuple or list
136 # If this attribute is a tuple or list
136 if type(inputObj.__dict__[key]) in (tuple, list):
137 if type(inputObj.__dict__[key]) in (tuple, list):
137 self.__dict__[key] = attribute[:]
138 self.__dict__[key] = attribute[:]
138 continue
139 continue
139
140
140 # If this attribute is another object or instance
141 # If this attribute is another object or instance
141 if hasattr(attribute, '__dict__'):
142 if hasattr(attribute, '__dict__'):
142 self.__dict__[key] = attribute.copy()
143 self.__dict__[key] = attribute.copy()
143 continue
144 continue
144
145
145 self.__dict__[key] = inputObj.__dict__[key]
146 self.__dict__[key] = inputObj.__dict__[key]
146
147
147 def deepcopy(self):
148 def deepcopy(self):
148
149
149 return copy.deepcopy(self)
150 return copy.deepcopy(self)
150
151
151 def isEmpty(self):
152 def isEmpty(self):
152
153
153 return self.flagNoData
154 return self.flagNoData
154
155
155 def isReady(self):
156 def isReady(self):
156
157
157 return not self.flagNoData
158 return not self.flagNoData
158
159
159
160
160 class JROData(GenericData):
161 class JROData(GenericData):
161
162
162 useInputBuffer = False
163 useInputBuffer = False
163 buffer_empty = True
164 buffer_empty = True
164
165
165 systemHeaderObj = SystemHeader()
166 systemHeaderObj = SystemHeader()
166 radarControllerHeaderObj = RadarControllerHeader()
167 radarControllerHeaderObj = RadarControllerHeader()
167 type = None
168 type = None
168 datatype = None # dtype but in string
169 datatype = None # dtype but in string
169 nProfiles = None
170 nProfiles = None
170 heightList = None
171 heightList = None
171 channelList = None
172 channelList = None
172 flagDiscontinuousBlock = False
173 flagDiscontinuousBlock = False
173 useLocalTime = False
174 useLocalTime = False
174 utctime = None
175 utctime = None
175 timeZone = None
176 timeZone = None
176 dstFlag = None
177 dstFlag = None
177 errorCount = None
178 errorCount = None
178 blocksize = None
179 blocksize = None
179 flagDecodeData = False # asumo q la data no esta decodificada
180 flagDecodeData = False # asumo q la data no esta decodificada
180 flagDeflipData = False # asumo q la data no esta sin flip
181 flagDeflipData = False # asumo q la data no esta sin flip
181 flagShiftFFT = False
182 flagShiftFFT = False
182 nCohInt = None
183 nCohInt = None
183 windowOfFilter = 1
184 windowOfFilter = 1
184 C = 3e8
185 C = 3e8
185 frequency = 49.92e6
186 frequency = 49.92e6
186 realtime = False
187 realtime = False
187 beacon_heiIndexList = None
188 beacon_heiIndexList = None
188 last_block = None
189 last_block = None
189 blocknow = None
190 blocknow = None
190 azimuth = None
191 azimuth = None
191 zenith = None
192 zenith = None
192 beam = Beam()
193 beam = Beam()
193 profileIndex = None
194 profileIndex = None
194 error = None
195 error = None
195 data = None
196 data = None
196 nmodes = None
197 nmodes = None
197 metadata_list = ['heightList', 'timeZone', 'type']
198 metadata_list = ['heightList', 'timeZone', 'type']
198 codeList = []
199 codeList = []
199 azimuthList = []
200 azimuthList = []
200 elevationList = []
201 elevationList = []
201
202
202 def __str__(self):
203 def __str__(self):
203
204
204 return '{} - {}'.format(self.type, self.datatime())
205 return '{} - {}'.format(self.type, self.datatime())
205
206
206 def getNoise(self):
207 def getNoise(self):
207
208
208 raise NotImplementedError
209 raise NotImplementedError
209
210
210 @property
211 @property
211 def nChannels(self):
212 def nChannels(self):
212
213
213 return len(self.channelList)
214 return len(self.channelList)
214
215
215 @property
216 @property
216 def channelIndexList(self):
217 def channelIndexList(self):
217
218
218 return list(range(self.nChannels))
219 return list(range(self.nChannels))
219
220
220 @property
221 @property
221 def nHeights(self):
222 def nHeights(self):
222
223
223 return len(self.heightList)
224 return len(self.heightList)
224
225
225 def getDeltaH(self):
226 def getDeltaH(self):
226
227
227 return self.heightList[1] - self.heightList[0]
228 return self.heightList[1] - self.heightList[0]
228
229
229 @property
230 @property
230 def ltctime(self):
231 def ltctime(self):
231
232
232 if self.useLocalTime:
233 if self.useLocalTime:
233 return self.utctime - self.timeZone * 60
234 return self.utctime - self.timeZone * 60
234
235
235 return self.utctime
236 return self.utctime
236
237
237 @property
238 @property
238 def datatime(self):
239 def datatime(self):
239
240
240 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
241 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
241 return datatimeValue
242 return datatimeValue
242
243
243 def getTimeRange(self):
244 def getTimeRange(self):
244
245
245 datatime = []
246 datatime = []
246
247
247 datatime.append(self.ltctime)
248 datatime.append(self.ltctime)
248 datatime.append(self.ltctime + self.timeInterval + 1)
249 datatime.append(self.ltctime + self.timeInterval + 1)
249
250
250 datatime = numpy.array(datatime)
251 datatime = numpy.array(datatime)
251
252
252 return datatime
253 return datatime
253
254
254 def getFmaxTimeResponse(self):
255 def getFmaxTimeResponse(self):
255
256
256 period = (10**-6) * self.getDeltaH() / (0.15)
257 period = (10**-6) * self.getDeltaH() / (0.15)
257
258
258 PRF = 1. / (period * self.nCohInt)
259 PRF = 1. / (period * self.nCohInt)
259
260
260 fmax = PRF
261 fmax = PRF
261
262
262 return fmax
263 return fmax
263
264
264 def getFmax(self):
265 def getFmax(self):
265 PRF = 1. / (self.ippSeconds * self.nCohInt)
266 PRF = 1. / (self.ippSeconds * self.nCohInt)
266
267
267 fmax = PRF
268 fmax = PRF
268 return fmax
269 return fmax
269
270
270 def getVmax(self):
271 def getVmax(self):
271
272
272 _lambda = self.C / self.frequency
273 _lambda = self.C / self.frequency
273
274
274 vmax = self.getFmax() * _lambda / 2
275 vmax = self.getFmax() * _lambda / 2
275
276
276 return vmax
277 return vmax
277
278
278 @property
279 @property
279 def ippSeconds(self):
280 def ippSeconds(self):
280 '''
281 '''
281 '''
282 '''
282 return self.radarControllerHeaderObj.ippSeconds
283 return self.radarControllerHeaderObj.ippSeconds
283
284
284 @ippSeconds.setter
285 @ippSeconds.setter
285 def ippSeconds(self, ippSeconds):
286 def ippSeconds(self, ippSeconds):
286 '''
287 '''
287 '''
288 '''
288 self.radarControllerHeaderObj.ippSeconds = ippSeconds
289 self.radarControllerHeaderObj.ippSeconds = ippSeconds
289
290
290 @property
291 @property
291 def code(self):
292 def code(self):
292 '''
293 '''
293 '''
294 '''
294 return self.radarControllerHeaderObj.code
295 return self.radarControllerHeaderObj.code
295
296
296 @code.setter
297 @code.setter
297 def code(self, code):
298 def code(self, code):
298 '''
299 '''
299 '''
300 '''
300 self.radarControllerHeaderObj.code = code
301 self.radarControllerHeaderObj.code = code
301
302
302 @property
303 @property
303 def nCode(self):
304 def nCode(self):
304 '''
305 '''
305 '''
306 '''
306 return self.radarControllerHeaderObj.nCode
307 return self.radarControllerHeaderObj.nCode
307
308
308 @nCode.setter
309 @nCode.setter
309 def nCode(self, ncode):
310 def nCode(self, ncode):
310 '''
311 '''
311 '''
312 '''
312 self.radarControllerHeaderObj.nCode = ncode
313 self.radarControllerHeaderObj.nCode = ncode
313
314
314 @property
315 @property
315 def nBaud(self):
316 def nBaud(self):
316 '''
317 '''
317 '''
318 '''
318 return self.radarControllerHeaderObj.nBaud
319 return self.radarControllerHeaderObj.nBaud
319
320
320 @nBaud.setter
321 @nBaud.setter
321 def nBaud(self, nbaud):
322 def nBaud(self, nbaud):
322 '''
323 '''
323 '''
324 '''
324 self.radarControllerHeaderObj.nBaud = nbaud
325 self.radarControllerHeaderObj.nBaud = nbaud
325
326
326 @property
327 @property
327 def ipp(self):
328 def ipp(self):
328 '''
329 '''
329 '''
330 '''
330 return self.radarControllerHeaderObj.ipp
331 return self.radarControllerHeaderObj.ipp
331
332
332 @ipp.setter
333 @ipp.setter
333 def ipp(self, ipp):
334 def ipp(self, ipp):
334 '''
335 '''
335 '''
336 '''
336 self.radarControllerHeaderObj.ipp = ipp
337 self.radarControllerHeaderObj.ipp = ipp
337
338
338 @property
339 @property
339 def metadata(self):
340 def metadata(self):
340 '''
341 '''
341 '''
342 '''
342
343
343 return {attr: getattr(self, attr) for attr in self.metadata_list}
344 return {attr: getattr(self, attr) for attr in self.metadata_list}
344
345
345
346
346 class Voltage(JROData):
347 class Voltage(JROData):
347
348
348 dataPP_POW = None
349 dataPP_POW = None
349 dataPP_DOP = None
350 dataPP_DOP = None
350 dataPP_WIDTH = None
351 dataPP_WIDTH = None
351 dataPP_SNR = None
352 dataPP_SNR = None
352
353
353 def __init__(self):
354 def __init__(self):
354 '''
355 '''
355 Constructor
356 Constructor
356 '''
357 '''
357
358
358 self.useLocalTime = True
359 self.useLocalTime = True
359 self.radarControllerHeaderObj = RadarControllerHeader()
360 self.radarControllerHeaderObj = RadarControllerHeader()
360 self.systemHeaderObj = SystemHeader()
361 self.systemHeaderObj = SystemHeader()
361 self.type = "Voltage"
362 self.type = "Voltage"
362 self.data = None
363 self.data = None
363 self.nProfiles = None
364 self.nProfiles = None
364 self.heightList = None
365 self.heightList = None
365 self.channelList = None
366 self.channelList = None
366 self.flagNoData = True
367 self.flagNoData = True
367 self.flagDiscontinuousBlock = False
368 self.flagDiscontinuousBlock = False
368 self.utctime = None
369 self.utctime = None
369 self.timeZone = 0
370 self.timeZone = 0
370 self.dstFlag = None
371 self.dstFlag = None
371 self.errorCount = None
372 self.errorCount = None
372 self.nCohInt = None
373 self.nCohInt = None
373 self.blocksize = None
374 self.blocksize = None
374 self.flagCohInt = False
375 self.flagCohInt = False
375 self.flagDecodeData = False # asumo q la data no esta decodificada
376 self.flagDecodeData = False # asumo q la data no esta decodificada
376 self.flagDeflipData = False # asumo q la data no esta sin flip
377 self.flagDeflipData = False # asumo q la data no esta sin flip
377 self.flagShiftFFT = False
378 self.flagShiftFFT = False
378 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
379 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
379 self.profileIndex = 0
380 self.profileIndex = 0
380 self.metadata_list = ['type', 'heightList', 'timeZone', 'nProfiles', 'channelList', 'nCohInt',
381 self.metadata_list = ['type', 'heightList', 'timeZone', 'nProfiles', 'channelList', 'nCohInt',
381 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp']
382 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp']
382
383
383 def getNoisebyHildebrand(self, channel=None):
384 def getNoisebyHildebrand(self, channel=None, ymin_index=None, ymax_index=None):
384 """
385 """
385 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
386 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
386
387
387 Return:
388 Return:
388 noiselevel
389 noiselevel
389 """
390 """
390
391
391 if channel != None:
392 if channel != None:
392 data = self.data[channel]
393 data = self.data[channel,ymin_index:ymax_index]
393 nChannels = 1
394 nChannels = 1
394 else:
395 else:
395 data = self.data
396 data = self.data[:,ymin_index:ymax_index]
396 nChannels = self.nChannels
397 nChannels = self.nChannels
397
398
398 noise = numpy.zeros(nChannels)
399 noise = numpy.zeros(nChannels)
399 power = data * numpy.conjugate(data)
400 power = data * numpy.conjugate(data)
400
401
401 for thisChannel in range(nChannels):
402 for thisChannel in range(nChannels):
402 if nChannels == 1:
403 if nChannels == 1:
403 daux = power[:].real
404 daux = power[:].real
404 else:
405 else:
405 daux = power[thisChannel, :].real
406 daux = power[thisChannel, :].real
406 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
407 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
407
408
408 return noise
409 return noise
409
410
410 def getNoise(self, type=1, channel=None):
411 def getNoise(self, type=1, channel=None,ymin_index=None, ymax_index=None):
411
412
412 if type == 1:
413 if type == 1:
413 noise = self.getNoisebyHildebrand(channel)
414 noise = self.getNoisebyHildebrand(channel,ymin_index, ymax_index)
414
415
415 return noise
416 return noise
416
417
417 def getPower(self, channel=None):
418 def getPower(self, channel=None):
418
419
419 if channel != None:
420 if channel != None:
420 data = self.data[channel]
421 data = self.data[channel]
421 else:
422 else:
422 data = self.data
423 data = self.data
423
424
424 power = data * numpy.conjugate(data)
425 power = data * numpy.conjugate(data)
425 powerdB = 10 * numpy.log10(power.real)
426 powerdB = 10 * numpy.log10(power.real)
426 powerdB = numpy.squeeze(powerdB)
427 powerdB = numpy.squeeze(powerdB)
427
428
428 return powerdB
429 return powerdB
429
430
430 @property
431 @property
431 def timeInterval(self):
432 def timeInterval(self):
432
433
433 return self.ippSeconds * self.nCohInt
434 return self.ippSeconds * self.nCohInt
434
435
435 noise = property(getNoise, "I'm the 'nHeights' property.")
436 noise = property(getNoise, "I'm the 'nHeights' property.")
436
437
437
438
438 class Spectra(JROData):
439 class Spectra(JROData):
439
440
441 data_outlier = 0
442
440 def __init__(self):
443 def __init__(self):
441 '''
444 '''
442 Constructor
445 Constructor
443 '''
446 '''
444
447
445 self.data_dc = None
448 self.data_dc = None
446 self.data_spc = None
449 self.data_spc = None
447 self.data_cspc = None
450 self.data_cspc = None
448 self.useLocalTime = True
451 self.useLocalTime = True
449 self.radarControllerHeaderObj = RadarControllerHeader()
452 self.radarControllerHeaderObj = RadarControllerHeader()
450 self.systemHeaderObj = SystemHeader()
453 self.systemHeaderObj = SystemHeader()
451 self.type = "Spectra"
454 self.type = "Spectra"
452 self.timeZone = 0
455 self.timeZone = 0
453 self.nProfiles = None
456 self.nProfiles = None
454 self.heightList = None
457 self.heightList = None
455 self.channelList = None
458 self.channelList = None
456 self.pairsList = None
459 self.pairsList = None
457 self.flagNoData = True
460 self.flagNoData = True
458 self.flagDiscontinuousBlock = False
461 self.flagDiscontinuousBlock = False
459 self.utctime = None
462 self.utctime = None
460 self.nCohInt = None
463 self.nCohInt = None
461 self.nIncohInt = None
464 self.nIncohInt = None
462 self.blocksize = None
465 self.blocksize = None
463 self.nFFTPoints = None
466 self.nFFTPoints = None
464 self.wavelength = None
467 self.wavelength = None
465 self.flagDecodeData = False # asumo q la data no esta decodificada
468 self.flagDecodeData = False # asumo q la data no esta decodificada
466 self.flagDeflipData = False # asumo q la data no esta sin flip
469 self.flagDeflipData = False # asumo q la data no esta sin flip
467 self.flagShiftFFT = False
470 self.flagShiftFFT = False
468 self.ippFactor = 1
471 self.ippFactor = 1
469 self.beacon_heiIndexList = []
472 self.beacon_heiIndexList = []
470 self.noise_estimation = None
473 self.noise_estimation = None
471 self.codeList = []
474 self.codeList = []
472 self.azimuthList = []
475 self.azimuthList = []
473 self.elevationList = []
476 self.elevationList = []
474 self.metadata_list = ['type', 'heightList', 'timeZone', 'pairsList', 'channelList', 'nCohInt',
477 self.metadata_list = ['type', 'heightList', 'timeZone', 'pairsList', 'channelList', 'nCohInt',
475 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp','nIncohInt', 'nFFTPoints', 'nProfiles']
478 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp','nIncohInt', 'nFFTPoints', 'nProfiles']
476
479
477
480
481 self.max_nIncohInt = 1
482
483
478 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
484 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
479 """
485 """
480 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
486 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
481
487
482 Return:
488 Return:
483 noiselevel
489 noiselevel
484 """
490 """
485
491 # if hasattr(self.nIncohInt, "__len__"): #nIncohInt is a matrix
492 #
493 # heis = self.data_spc.shape[2]
494 #
495 # noise = numpy.zeros((self.nChannels, heis))
496 # for hei in range(heis):
497 # for channel in range(self.nChannels):
498 # daux = self.data_spc[channel, xmin_index:xmax_index, hei]
499 #
500 # noise[channel,hei] = hildebrand_sekhon(daux, self.nIncohInt[channel,hei])
501 #
502 # else:
503 # noise = numpy.zeros(self.nChannels)
504 # for channel in range(self.nChannels):
505 # daux = self.data_spc[channel,xmin_index:xmax_index, ymin_index:ymax_index]
506 #
507 # noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
486 noise = numpy.zeros(self.nChannels)
508 noise = numpy.zeros(self.nChannels)
487 for channel in range(self.nChannels):
509 for channel in range(self.nChannels):
488 daux = self.data_spc[channel,
510 daux = self.data_spc[channel,xmin_index:xmax_index, ymin_index:ymax_index]
489 xmin_index:xmax_index, ymin_index:ymax_index]
511
490 noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
512 noise[channel] = hildebrand_sekhon(daux, self.max_nIncohInt)
491
513
492 return noise
514 return noise
493
515
494 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
516 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
495
517
496 if self.noise_estimation is not None:
518 if self.noise_estimation is not None:
497 # this was estimated by getNoise Operation defined in jroproc_spectra.py
519 # this was estimated by getNoise Operation defined in jroproc_spectra.py
498 return self.noise_estimation
520 return self.noise_estimation
499 else:
521 else:
500 noise = self.getNoisebyHildebrand(
522 noise = self.getNoisebyHildebrand(
501 xmin_index, xmax_index, ymin_index, ymax_index)
523 xmin_index, xmax_index, ymin_index, ymax_index)
502 return noise
524 return noise
503
525
504 def getFreqRangeTimeResponse(self, extrapoints=0):
526 def getFreqRangeTimeResponse(self, extrapoints=0):
505
527
506 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
528 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
507 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.) - deltafreq / 2
529 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.) - deltafreq / 2
508
530
509 return freqrange
531 return freqrange
510
532
511 def getAcfRange(self, extrapoints=0):
533 def getAcfRange(self, extrapoints=0):
512
534
513 deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
535 deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
514 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
536 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
515
537
516 return freqrange
538 return freqrange
517
539
518 def getFreqRange(self, extrapoints=0):
540 def getFreqRange(self, extrapoints=0):
519
541
520 deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
542 deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
521 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
543 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
522
544
523 return freqrange
545 return freqrange
524
546
525 def getVelRange(self, extrapoints=0):
547 def getVelRange(self, extrapoints=0):
526
548
527 deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
549 deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
528 velrange = deltav * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.)
550 velrange = deltav * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.)
529
551
530 if self.nmodes:
552 if self.nmodes:
531 return velrange/self.nmodes
553 return velrange/self.nmodes
532 else:
554 else:
533 return velrange
555 return velrange
534
556
535 @property
557 @property
536 def nPairs(self):
558 def nPairs(self):
537
559
538 return len(self.pairsList)
560 return len(self.pairsList)
539
561
540 @property
562 @property
541 def pairsIndexList(self):
563 def pairsIndexList(self):
542
564
543 return list(range(self.nPairs))
565 return list(range(self.nPairs))
544
566
545 @property
567 @property
546 def normFactor(self):
568 def normFactor(self):
547
569
548 pwcode = 1
570 pwcode = 1
549
571
550 if self.flagDecodeData:
572 if self.flagDecodeData:
551 pwcode = numpy.sum(self.code[0]**2)
573 pwcode = numpy.sum(self.code[0]**2)
552 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
574 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
553 normFactor = self.nProfiles * self.nIncohInt * self.nCohInt * pwcode * self.windowOfFilter
575 normFactor = self.nProfiles * self.nIncohInt * self.nCohInt * pwcode * self.windowOfFilter
554
576
577
555 return normFactor
578 return normFactor
556
579
557 @property
580 @property
558 def flag_cspc(self):
581 def flag_cspc(self):
559
582
560 if self.data_cspc is None:
583 if self.data_cspc is None:
561 return True
584 return True
562
585
563 return False
586 return False
564
587
565 @property
588 @property
566 def flag_dc(self):
589 def flag_dc(self):
567
590
568 if self.data_dc is None:
591 if self.data_dc is None:
569 return True
592 return True
570
593
571 return False
594 return False
572
595
573 @property
596 @property
574 def timeInterval(self):
597 def timeInterval(self):
575
598
576 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
599 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
577 if self.nmodes:
600 if self.nmodes:
578 return self.nmodes*timeInterval
601 return self.nmodes*timeInterval
579 else:
602 else:
580 return timeInterval
603 return timeInterval
581
604
582 def getPower(self):
605 def getPower(self):
583
606
584 factor = self.normFactor
607 factor = self.normFactor
585 z = self.data_spc / factor
608 z = numpy.divide(self.data_spc,factor)
586 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
609 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
587 avg = numpy.average(z, axis=1)
610 avg = numpy.average(z, axis=1)
588
611
589 return 10 * numpy.log10(avg)
612 return 10 * numpy.log10(avg)
590
613
591 def getCoherence(self, pairsList=None, phase=False):
614 def getCoherence(self, pairsList=None, phase=False):
592
615
593 z = []
616 z = []
594 if pairsList is None:
617 if pairsList is None:
595 pairsIndexList = self.pairsIndexList
618 pairsIndexList = self.pairsIndexList
596 else:
619 else:
597 pairsIndexList = []
620 pairsIndexList = []
598 for pair in pairsList:
621 for pair in pairsList:
599 if pair not in self.pairsList:
622 if pair not in self.pairsList:
600 raise ValueError("Pair %s is not in dataOut.pairsList" % (
623 raise ValueError("Pair %s is not in dataOut.pairsList" % (
601 pair))
624 pair))
602 pairsIndexList.append(self.pairsList.index(pair))
625 pairsIndexList.append(self.pairsList.index(pair))
603 for i in range(len(pairsIndexList)):
626 for i in range(len(pairsIndexList)):
604 pair = self.pairsList[pairsIndexList[i]]
627 pair = self.pairsList[pairsIndexList[i]]
605 ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
628 ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
606 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
629 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
607 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
630 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
608 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
631 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
609 if phase:
632 if phase:
610 data = numpy.arctan2(avgcoherenceComplex.imag,
633 data = numpy.arctan2(avgcoherenceComplex.imag,
611 avgcoherenceComplex.real) * 180 / numpy.pi
634 avgcoherenceComplex.real) * 180 / numpy.pi
612 else:
635 else:
613 data = numpy.abs(avgcoherenceComplex)
636 data = numpy.abs(avgcoherenceComplex)
614
637
615 z.append(data)
638 z.append(data)
616
639
617 return numpy.array(z)
640 return numpy.array(z)
618
641
619 def setValue(self, value):
642 def setValue(self, value):
620
643
621 print("This property should not be initialized")
644 print("This property should not be initialized")
622
645
623 return
646 return
624
647
625 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
648 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
626
649
627
650
628 class SpectraHeis(Spectra):
651 class SpectraHeis(Spectra):
629
652
630 def __init__(self):
653 def __init__(self):
631
654
632 self.radarControllerHeaderObj = RadarControllerHeader()
655 self.radarControllerHeaderObj = RadarControllerHeader()
633 self.systemHeaderObj = SystemHeader()
656 self.systemHeaderObj = SystemHeader()
634 self.type = "SpectraHeis"
657 self.type = "SpectraHeis"
635 self.nProfiles = None
658 self.nProfiles = None
636 self.heightList = None
659 self.heightList = None
637 self.channelList = None
660 self.channelList = None
638 self.flagNoData = True
661 self.flagNoData = True
639 self.flagDiscontinuousBlock = False
662 self.flagDiscontinuousBlock = False
640 self.utctime = None
663 self.utctime = None
641 self.blocksize = None
664 self.blocksize = None
642 self.profileIndex = 0
665 self.profileIndex = 0
643 self.nCohInt = 1
666 self.nCohInt = 1
644 self.nIncohInt = 1
667 self.nIncohInt = 1
645
668
646 @property
669 @property
647 def normFactor(self):
670 def normFactor(self):
648 pwcode = 1
671 pwcode = 1
649 if self.flagDecodeData:
672 if self.flagDecodeData:
650 pwcode = numpy.sum(self.code[0]**2)
673 pwcode = numpy.sum(self.code[0]**2)
651
674
652 normFactor = self.nIncohInt * self.nCohInt * pwcode
675 normFactor = self.nIncohInt * self.nCohInt * pwcode
653
676
654 return normFactor
677 return normFactor
655
678
656 @property
679 @property
657 def timeInterval(self):
680 def timeInterval(self):
658
681
659 return self.ippSeconds * self.nCohInt * self.nIncohInt
682 return self.ippSeconds * self.nCohInt * self.nIncohInt
660
683
661
684
662 class Fits(JROData):
685 class Fits(JROData):
663
686
664 def __init__(self):
687 def __init__(self):
665
688
666 self.type = "Fits"
689 self.type = "Fits"
667 self.nProfiles = None
690 self.nProfiles = None
668 self.heightList = None
691 self.heightList = None
669 self.channelList = None
692 self.channelList = None
670 self.flagNoData = True
693 self.flagNoData = True
671 self.utctime = None
694 self.utctime = None
672 self.nCohInt = 1
695 self.nCohInt = 1
673 self.nIncohInt = 1
696 self.nIncohInt = 1
674 self.useLocalTime = True
697 self.useLocalTime = True
675 self.profileIndex = 0
698 self.profileIndex = 0
676 self.timeZone = 0
699 self.timeZone = 0
677
700
678 def getTimeRange(self):
701 def getTimeRange(self):
679
702
680 datatime = []
703 datatime = []
681
704
682 datatime.append(self.ltctime)
705 datatime.append(self.ltctime)
683 datatime.append(self.ltctime + self.timeInterval)
706 datatime.append(self.ltctime + self.timeInterval)
684
707
685 datatime = numpy.array(datatime)
708 datatime = numpy.array(datatime)
686
709
687 return datatime
710 return datatime
688
711
689 def getChannelIndexList(self):
712 def getChannelIndexList(self):
690
713
691 return list(range(self.nChannels))
714 return list(range(self.nChannels))
692
715
693 def getNoise(self, type=1):
716 def getNoise(self, type=1):
694
717
695
718
696 if type == 1:
719 if type == 1:
697 noise = self.getNoisebyHildebrand()
720 noise = self.getNoisebyHildebrand()
698
721
699 if type == 2:
722 if type == 2:
700 noise = self.getNoisebySort()
723 noise = self.getNoisebySort()
701
724
702 if type == 3:
725 if type == 3:
703 noise = self.getNoisebyWindow()
726 noise = self.getNoisebyWindow()
704
727
705 return noise
728 return noise
706
729
707 @property
730 @property
708 def timeInterval(self):
731 def timeInterval(self):
709
732
710 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
733 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
711
734
712 return timeInterval
735 return timeInterval
713
736
714 @property
737 @property
715 def ippSeconds(self):
738 def ippSeconds(self):
716 '''
739 '''
717 '''
740 '''
718 return self.ipp_sec
741 return self.ipp_sec
719
742
720 noise = property(getNoise, "I'm the 'nHeights' property.")
743 noise = property(getNoise, "I'm the 'nHeights' property.")
721
744
722
745
723 class Correlation(JROData):
746 class Correlation(JROData):
724
747
725 def __init__(self):
748 def __init__(self):
726 '''
749 '''
727 Constructor
750 Constructor
728 '''
751 '''
729 self.radarControllerHeaderObj = RadarControllerHeader()
752 self.radarControllerHeaderObj = RadarControllerHeader()
730 self.systemHeaderObj = SystemHeader()
753 self.systemHeaderObj = SystemHeader()
731 self.type = "Correlation"
754 self.type = "Correlation"
732 self.data = None
755 self.data = None
733 self.dtype = None
756 self.dtype = None
734 self.nProfiles = None
757 self.nProfiles = None
735 self.heightList = None
758 self.heightList = None
736 self.channelList = None
759 self.channelList = None
737 self.flagNoData = True
760 self.flagNoData = True
738 self.flagDiscontinuousBlock = False
761 self.flagDiscontinuousBlock = False
739 self.utctime = None
762 self.utctime = None
740 self.timeZone = 0
763 self.timeZone = 0
741 self.dstFlag = None
764 self.dstFlag = None
742 self.errorCount = None
765 self.errorCount = None
743 self.blocksize = None
766 self.blocksize = None
744 self.flagDecodeData = False # asumo q la data no esta decodificada
767 self.flagDecodeData = False # asumo q la data no esta decodificada
745 self.flagDeflipData = False # asumo q la data no esta sin flip
768 self.flagDeflipData = False # asumo q la data no esta sin flip
746 self.pairsList = None
769 self.pairsList = None
747 self.nPoints = None
770 self.nPoints = None
748
771
749 def getPairsList(self):
772 def getPairsList(self):
750
773
751 return self.pairsList
774 return self.pairsList
752
775
753 def getNoise(self, mode=2):
776 def getNoise(self, mode=2):
754
777
755 indR = numpy.where(self.lagR == 0)[0][0]
778 indR = numpy.where(self.lagR == 0)[0][0]
756 indT = numpy.where(self.lagT == 0)[0][0]
779 indT = numpy.where(self.lagT == 0)[0][0]
757
780
758 jspectra0 = self.data_corr[:, :, indR, :]
781 jspectra0 = self.data_corr[:, :, indR, :]
759 jspectra = copy.copy(jspectra0)
782 jspectra = copy.copy(jspectra0)
760
783
761 num_chan = jspectra.shape[0]
784 num_chan = jspectra.shape[0]
762 num_hei = jspectra.shape[2]
785 num_hei = jspectra.shape[2]
763
786
764 freq_dc = jspectra.shape[1] / 2
787 freq_dc = jspectra.shape[1] / 2
765 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
788 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
766
789
767 if ind_vel[0] < 0:
790 if ind_vel[0] < 0:
768 ind_vel[list(range(0, 1))] = ind_vel[list(
791 ind_vel[list(range(0, 1))] = ind_vel[list(
769 range(0, 1))] + self.num_prof
792 range(0, 1))] + self.num_prof
770
793
771 if mode == 1:
794 if mode == 1:
772 jspectra[:, freq_dc, :] = (
795 jspectra[:, freq_dc, :] = (
773 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
796 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
774
797
775 if mode == 2:
798 if mode == 2:
776
799
777 vel = numpy.array([-2, -1, 1, 2])
800 vel = numpy.array([-2, -1, 1, 2])
778 xx = numpy.zeros([4, 4])
801 xx = numpy.zeros([4, 4])
779
802
780 for fil in range(4):
803 for fil in range(4):
781 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
804 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
782
805
783 xx_inv = numpy.linalg.inv(xx)
806 xx_inv = numpy.linalg.inv(xx)
784 xx_aux = xx_inv[0, :]
807 xx_aux = xx_inv[0, :]
785
808
786 for ich in range(num_chan):
809 for ich in range(num_chan):
787 yy = jspectra[ich, ind_vel, :]
810 yy = jspectra[ich, ind_vel, :]
788 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
811 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
789
812
790 junkid = jspectra[ich, freq_dc, :] <= 0
813 junkid = jspectra[ich, freq_dc, :] <= 0
791 cjunkid = sum(junkid)
814 cjunkid = sum(junkid)
792
815
793 if cjunkid.any():
816 if cjunkid.any():
794 jspectra[ich, freq_dc, junkid.nonzero()] = (
817 jspectra[ich, freq_dc, junkid.nonzero()] = (
795 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
818 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
796
819
797 noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
820 noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
798
821
799 return noise
822 return noise
800
823
801 @property
824 @property
802 def timeInterval(self):
825 def timeInterval(self):
803
826
804 return self.ippSeconds * self.nCohInt * self.nProfiles
827 return self.ippSeconds * self.nCohInt * self.nProfiles
805
828
806 def splitFunctions(self):
829 def splitFunctions(self):
807
830
808 pairsList = self.pairsList
831 pairsList = self.pairsList
809 ccf_pairs = []
832 ccf_pairs = []
810 acf_pairs = []
833 acf_pairs = []
811 ccf_ind = []
834 ccf_ind = []
812 acf_ind = []
835 acf_ind = []
813 for l in range(len(pairsList)):
836 for l in range(len(pairsList)):
814 chan0 = pairsList[l][0]
837 chan0 = pairsList[l][0]
815 chan1 = pairsList[l][1]
838 chan1 = pairsList[l][1]
816
839
817 # Obteniendo pares de Autocorrelacion
840 # Obteniendo pares de Autocorrelacion
818 if chan0 == chan1:
841 if chan0 == chan1:
819 acf_pairs.append(chan0)
842 acf_pairs.append(chan0)
820 acf_ind.append(l)
843 acf_ind.append(l)
821 else:
844 else:
822 ccf_pairs.append(pairsList[l])
845 ccf_pairs.append(pairsList[l])
823 ccf_ind.append(l)
846 ccf_ind.append(l)
824
847
825 data_acf = self.data_cf[acf_ind]
848 data_acf = self.data_cf[acf_ind]
826 data_ccf = self.data_cf[ccf_ind]
849 data_ccf = self.data_cf[ccf_ind]
827
850
828 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
851 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
829
852
830 @property
853 @property
831 def normFactor(self):
854 def normFactor(self):
832 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
855 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
833 acf_pairs = numpy.array(acf_pairs)
856 acf_pairs = numpy.array(acf_pairs)
834 normFactor = numpy.zeros((self.nPairs, self.nHeights))
857 normFactor = numpy.zeros((self.nPairs, self.nHeights))
835
858
836 for p in range(self.nPairs):
859 for p in range(self.nPairs):
837 pair = self.pairsList[p]
860 pair = self.pairsList[p]
838
861
839 ch0 = pair[0]
862 ch0 = pair[0]
840 ch1 = pair[1]
863 ch1 = pair[1]
841
864
842 ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
865 ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
843 ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
866 ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
844 normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
867 normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
845
868
846 return normFactor
869 return normFactor
847
870
848
871
849 class Parameters(Spectra):
872 class Parameters(Spectra):
850
873
851 groupList = None # List of Pairs, Groups, etc
874 groupList = None # List of Pairs, Groups, etc
852 data_param = None # Parameters obtained
875 data_param = None # Parameters obtained
853 data_pre = None # Data Pre Parametrization
876 data_pre = None # Data Pre Parametrization
854 data_SNR = None # Signal to Noise Ratio
877 data_SNR = None # Signal to Noise Ratio
855 abscissaList = None # Abscissa, can be velocities, lags or time
878 abscissaList = None # Abscissa, can be velocities, lags or time
856 utctimeInit = None # Initial UTC time
879 utctimeInit = None # Initial UTC time
857 paramInterval = None # Time interval to calculate Parameters in seconds
880 paramInterval = None # Time interval to calculate Parameters in seconds
858 useLocalTime = True
881 useLocalTime = True
859 # Fitting
882 # Fitting
860 data_error = None # Error of the estimation
883 data_error = None # Error of the estimation
861 constants = None
884 constants = None
862 library = None
885 library = None
863 # Output signal
886 # Output signal
864 outputInterval = None # Time interval to calculate output signal in seconds
887 outputInterval = None # Time interval to calculate output signal in seconds
865 data_output = None # Out signal
888 data_output = None # Out signal
866 nAvg = None
889 nAvg = None
867 noise_estimation = None
890 noise_estimation = None
868 GauSPC = None # Fit gaussian SPC
891 GauSPC = None # Fit gaussian SPC
869
892
870 def __init__(self):
893 def __init__(self):
871 '''
894 '''
872 Constructor
895 Constructor
873 '''
896 '''
874 self.radarControllerHeaderObj = RadarControllerHeader()
897 self.radarControllerHeaderObj = RadarControllerHeader()
875 self.systemHeaderObj = SystemHeader()
898 self.systemHeaderObj = SystemHeader()
876 self.type = "Parameters"
899 self.type = "Parameters"
877 self.timeZone = 0
900 self.timeZone = 0
878
901
879 def getTimeRange1(self, interval):
902 def getTimeRange1(self, interval):
880
903
881 datatime = []
904 datatime = []
882
905
883 if self.useLocalTime:
906 if self.useLocalTime:
884 time1 = self.utctimeInit - self.timeZone * 60
907 time1 = self.utctimeInit - self.timeZone * 60
885 else:
908 else:
886 time1 = self.utctimeInit
909 time1 = self.utctimeInit
887
910
888 datatime.append(time1)
911 datatime.append(time1)
889 datatime.append(time1 + interval)
912 datatime.append(time1 + interval)
890 datatime = numpy.array(datatime)
913 datatime = numpy.array(datatime)
891
914
892 return datatime
915 return datatime
893
916
894 @property
917 @property
895 def timeInterval(self):
918 def timeInterval(self):
896
919
897 if hasattr(self, 'timeInterval1'):
920 if hasattr(self, 'timeInterval1'):
898 return self.timeInterval1
921 return self.timeInterval1
899 else:
922 else:
900 return self.paramInterval
923 return self.paramInterval
901
924
902 def setValue(self, value):
925 def setValue(self, value):
903
926
904 print("This property should not be initialized")
927 print("This property should not be initialized")
905
928
906 return
929 return
907
930
908 def getNoise(self):
931 def getNoise(self):
909
932
910 return self.spc_noise
933 return self.spc_noise
911
934
912 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
935 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
913
936
914
937
915 class PlotterData(object):
938 class PlotterData(object):
916 '''
939 '''
917 Object to hold data to be plotted
940 Object to hold data to be plotted
918 '''
941 '''
919
942
920 MAXNUMX = 200
943 MAXNUMX = 200
921 MAXNUMY = 200
944 MAXNUMY = 200
922
945
923 def __init__(self, code, exp_code, localtime=True):
946 def __init__(self, code, exp_code, localtime=True):
924
947
925 self.key = code
948 self.key = code
926 self.exp_code = exp_code
949 self.exp_code = exp_code
927 self.ready = False
950 self.ready = False
928 self.flagNoData = False
951 self.flagNoData = False
929 self.localtime = localtime
952 self.localtime = localtime
930 self.data = {}
953 self.data = {}
931 self.meta = {}
954 self.meta = {}
932 self.__heights = []
955 self.__heights = []
933
956
934 def __str__(self):
957 def __str__(self):
935 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
958 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
936 return 'Data[{}][{}]'.format(';'.join(dum), len(self.times))
959 return 'Data[{}][{}]'.format(';'.join(dum), len(self.times))
937
960
938 def __len__(self):
961 def __len__(self):
939 return len(self.data)
962 return len(self.data)
940
963
941 def __getitem__(self, key):
964 def __getitem__(self, key):
942 if isinstance(key, int):
965 if isinstance(key, int):
943 return self.data[self.times[key]]
966 return self.data[self.times[key]]
944 elif isinstance(key, str):
967 elif isinstance(key, str):
945 ret = numpy.array([self.data[x][key] for x in self.times])
968 ret = numpy.array([self.data[x][key] for x in self.times])
946 if ret.ndim > 1:
969 if ret.ndim > 1:
947 ret = numpy.swapaxes(ret, 0, 1)
970 ret = numpy.swapaxes(ret, 0, 1)
948 return ret
971 return ret
949
972
950 def __contains__(self, key):
973 def __contains__(self, key):
951 return key in self.data[self.min_time]
974 return key in self.data[self.min_time]
952
975
953 def setup(self):
976 def setup(self):
954 '''
977 '''
955 Configure object
978 Configure object
956 '''
979 '''
957 self.type = ''
980 self.type = ''
958 self.ready = False
981 self.ready = False
959 del self.data
982 del self.data
960 self.data = {}
983 self.data = {}
961 self.__heights = []
984 self.__heights = []
962 self.__all_heights = set()
985 self.__all_heights = set()
963
986
964 def shape(self, key):
987 def shape(self, key):
965 '''
988 '''
966 Get the shape of the one-element data for the given key
989 Get the shape of the one-element data for the given key
967 '''
990 '''
968
991
969 if len(self.data[self.min_time][key]):
992 if len(self.data[self.min_time][key]):
970 return self.data[self.min_time][key].shape
993 return self.data[self.min_time][key].shape
971 return (0,)
994 return (0,)
972
995
973 def update(self, data, tm, meta={}):
996 def update(self, data, tm, meta={}):
974 '''
997 '''
975 Update data object with new dataOut
998 Update data object with new dataOut
976 '''
999 '''
977
1000
978 self.data[tm] = data
1001 self.data[tm] = data
979
1002
980 for key, value in meta.items():
1003 for key, value in meta.items():
981 setattr(self, key, value)
1004 setattr(self, key, value)
982
1005
983 def normalize_heights(self):
1006 def normalize_heights(self):
984 '''
1007 '''
985 Ensure same-dimension of the data for different heighList
1008 Ensure same-dimension of the data for different heighList
986 '''
1009 '''
987
1010
988 H = numpy.array(list(self.__all_heights))
1011 H = numpy.array(list(self.__all_heights))
989 H.sort()
1012 H.sort()
990 for key in self.data:
1013 for key in self.data:
991 shape = self.shape(key)[:-1] + H.shape
1014 shape = self.shape(key)[:-1] + H.shape
992 for tm, obj in list(self.data[key].items()):
1015 for tm, obj in list(self.data[key].items()):
993 h = self.__heights[self.times.tolist().index(tm)]
1016 h = self.__heights[self.times.tolist().index(tm)]
994 if H.size == h.size:
1017 if H.size == h.size:
995 continue
1018 continue
996 index = numpy.where(numpy.in1d(H, h))[0]
1019 index = numpy.where(numpy.in1d(H, h))[0]
997 dummy = numpy.zeros(shape) + numpy.nan
1020 dummy = numpy.zeros(shape) + numpy.nan
998 if len(shape) == 2:
1021 if len(shape) == 2:
999 dummy[:, index] = obj
1022 dummy[:, index] = obj
1000 else:
1023 else:
1001 dummy[index] = obj
1024 dummy[index] = obj
1002 self.data[key][tm] = dummy
1025 self.data[key][tm] = dummy
1003
1026
1004 self.__heights = [H for tm in self.times]
1027 self.__heights = [H for tm in self.times]
1005
1028
1006 def jsonify(self, tm, plot_name, plot_type, decimate=False):
1029 def jsonify(self, tm, plot_name, plot_type, decimate=False):
1007 '''
1030 '''
1008 Convert data to json
1031 Convert data to json
1009 '''
1032 '''
1010
1033
1011 meta = {}
1034 meta = {}
1012 meta['xrange'] = []
1035 meta['xrange'] = []
1013 dy = int(len(self.yrange)/self.MAXNUMY) + 1
1036 dy = int(len(self.yrange)/self.MAXNUMY) + 1
1014 tmp = self.data[tm][self.key]
1037 tmp = self.data[tm][self.key]
1015 shape = tmp.shape
1038 shape = tmp.shape
1016 if len(shape) == 2:
1039 if len(shape) == 2:
1017 data = self.roundFloats(self.data[tm][self.key][::, ::dy].tolist())
1040 data = self.roundFloats(self.data[tm][self.key][::, ::dy].tolist())
1018 elif len(shape) == 3:
1041 elif len(shape) == 3:
1019 dx = int(self.data[tm][self.key].shape[1]/self.MAXNUMX) + 1
1042 dx = int(self.data[tm][self.key].shape[1]/self.MAXNUMX) + 1
1020 data = self.roundFloats(
1043 data = self.roundFloats(
1021 self.data[tm][self.key][::, ::dx, ::dy].tolist())
1044 self.data[tm][self.key][::, ::dx, ::dy].tolist())
1022 meta['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
1045 meta['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
1023 else:
1046 else:
1024 data = self.roundFloats(self.data[tm][self.key].tolist())
1047 data = self.roundFloats(self.data[tm][self.key].tolist())
1025
1048
1026 ret = {
1049 ret = {
1027 'plot': plot_name,
1050 'plot': plot_name,
1028 'code': self.exp_code,
1051 'code': self.exp_code,
1029 'time': float(tm),
1052 'time': float(tm),
1030 'data': data,
1053 'data': data,
1031 }
1054 }
1032 meta['type'] = plot_type
1055 meta['type'] = plot_type
1033 meta['interval'] = float(self.interval)
1056 meta['interval'] = float(self.interval)
1034 meta['localtime'] = self.localtime
1057 meta['localtime'] = self.localtime
1035 meta['yrange'] = self.roundFloats(self.yrange[::dy].tolist())
1058 meta['yrange'] = self.roundFloats(self.yrange[::dy].tolist())
1036 meta.update(self.meta)
1059 meta.update(self.meta)
1037 ret['metadata'] = meta
1060 ret['metadata'] = meta
1038 return json.dumps(ret)
1061 return json.dumps(ret)
1039
1062
1040 @property
1063 @property
1041 def times(self):
1064 def times(self):
1042 '''
1065 '''
1043 Return the list of times of the current data
1066 Return the list of times of the current data
1044 '''
1067 '''
1045
1068
1046 ret = [t for t in self.data]
1069 ret = [t for t in self.data]
1047 ret.sort()
1070 ret.sort()
1048 return numpy.array(ret)
1071 return numpy.array(ret)
1049
1072
1050 @property
1073 @property
1051 def min_time(self):
1074 def min_time(self):
1052 '''
1075 '''
1053 Return the minimun time value
1076 Return the minimun time value
1054 '''
1077 '''
1055
1078
1056 return self.times[0]
1079 return self.times[0]
1057
1080
1058 @property
1081 @property
1059 def max_time(self):
1082 def max_time(self):
1060 '''
1083 '''
1061 Return the maximun time value
1084 Return the maximun time value
1062 '''
1085 '''
1063
1086
1064 return self.times[-1]
1087 return self.times[-1]
1065
1088
1066 # @property
1089 # @property
1067 # def heights(self):
1090 # def heights(self):
1068 # '''
1091 # '''
1069 # Return the list of heights of the current data
1092 # Return the list of heights of the current data
1070 # '''
1093 # '''
1071
1094
1072 # return numpy.array(self.__heights[-1])
1095 # return numpy.array(self.__heights[-1])
1073
1096
1074 @staticmethod
1097 @staticmethod
1075 def roundFloats(obj):
1098 def roundFloats(obj):
1076 if isinstance(obj, list):
1099 if isinstance(obj, list):
1077 return list(map(PlotterData.roundFloats, obj))
1100 return list(map(PlotterData.roundFloats, obj))
1078 elif isinstance(obj, float):
1101 elif isinstance(obj, float):
1079 return round(obj, 2)
1102 return round(obj, 2)
Show file before | Show file after | Hide comments
@@ -1,1016 +1,1180
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """Classes to plot Spectra data
5 """Classes to plot Spectra data
6
6
7 """
7 """
8
8
9 import os
9 import os
10 import numpy
10 import numpy
11
11
12 from schainpy.model.graphics.jroplot_base import Plot, plt, log
12 from schainpy.model.graphics.jroplot_base import Plot, plt, log
13 from itertools import combinations
13 from itertools import combinations
14 from matplotlib.ticker import LinearLocator
14 from matplotlib.ticker import LinearLocator
15
15
16 class SpectraPlot(Plot):
16 class SpectraPlot(Plot):
17 '''
17 '''
18 Plot for Spectra data
18 Plot for Spectra data
19 '''
19 '''
20
20
21 CODE = 'spc'
21 CODE = 'spc'
22 colormap = 'jet'
22 colormap = 'jet'
23 plot_type = 'pcolor'
23 plot_type = 'pcolor'
24 buffering = False
24 buffering = False
25 channelList = []
25 channelList = []
26 elevationList = []
26 elevationList = []
27 azimuthList = []
27 azimuthList = []
28
28
29 def setup(self):
29 def setup(self):
30
30
31 self.nplots = len(self.data.channels)
31 self.nplots = len(self.data.channels)
32 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
32 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
33 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
33 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
34 self.height = 3.4 * self.nrows
34 self.height = 3.4 * self.nrows
35
35
36 self.cb_label = 'dB'
36 self.cb_label = 'dB'
37 if self.showprofile:
37 if self.showprofile:
38 self.width = 5.2 * self.ncols
38 self.width = 5.2 * self.ncols
39 else:
39 else:
40 self.width = 4.2* self.ncols
40 self.width = 4.2* self.ncols
41 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.12})
41 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.12})
42 self.ylabel = 'Range [km]'
42 self.ylabel = 'Range [km]'
43
43
44
44
45 def update_list(self,dataOut):
45 def update_list(self,dataOut):
46 if len(self.channelList) == 0:
46 if len(self.channelList) == 0:
47 self.channelList = dataOut.channelList
47 self.channelList = dataOut.channelList
48 if len(self.elevationList) == 0:
48 if len(self.elevationList) == 0:
49 self.elevationList = dataOut.elevationList
49 self.elevationList = dataOut.elevationList
50 if len(self.azimuthList) == 0:
50 if len(self.azimuthList) == 0:
51 self.azimuthList = dataOut.azimuthList
51 self.azimuthList = dataOut.azimuthList
52
52
53 def update(self, dataOut):
53 def update(self, dataOut):
54
54
55 self.update_list(dataOut)
55 self.update_list(dataOut)
56 data = {}
56 data = {}
57 meta = {}
57 meta = {}
58 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
58 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
59 data['spc'] = spc
59 data['spc'] = spc
60 data['rti'] = dataOut.getPower()
60 data['rti'] = dataOut.getPower()
61 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
61 #data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
62 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
62 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
63 if self.CODE == 'spc_moments':
63 if self.CODE == 'spc_moments':
64 data['moments'] = dataOut.moments
64 data['moments'] = dataOut.moments
65
65
66 return data, meta
66 return data, meta
67
67
68 def plot(self):
68 def plot(self):
69 if self.xaxis == "frequency":
69 if self.xaxis == "frequency":
70 x = self.data.xrange[0]
70 x = self.data.xrange[0]
71 self.xlabel = "Frequency (kHz)"
71 self.xlabel = "Frequency (kHz)"
72 elif self.xaxis == "time":
72 elif self.xaxis == "time":
73 x = self.data.xrange[1]
73 x = self.data.xrange[1]
74 self.xlabel = "Time (ms)"
74 self.xlabel = "Time (ms)"
75 else:
75 else:
76 x = self.data.xrange[2]
76 x = self.data.xrange[2]
77 self.xlabel = "Velocity (m/s)"
77 self.xlabel = "Velocity (m/s)"
78
78
79 if self.CODE == 'spc_moments':
79 if self.CODE == 'spc_moments':
80 x = self.data.xrange[2]
80 x = self.data.xrange[2]
81 self.xlabel = "Velocity (m/s)"
81 self.xlabel = "Velocity (m/s)"
82
82
83 self.titles = []
83 self.titles = []
84 y = self.data.yrange
84 y = self.data.yrange
85 self.y = y
85 self.y = y
86
86
87 data = self.data[-1]
87 data = self.data[-1]
88 z = data['spc']
88 z = data['spc']
89 print(z.shape, x.shape, y.shape)
89 #print(z.shape, x.shape, y.shape)
90 for n, ax in enumerate(self.axes):
90 for n, ax in enumerate(self.axes):
91 noise = data['noise'][n]
91 #noise = data['noise'][n]
92 noise=62
92 if self.CODE == 'spc_moments':
93 if self.CODE == 'spc_moments':
93 mean = data['moments'][n, 1]
94 mean = data['moments'][n, 1]
94 if ax.firsttime:
95 if ax.firsttime:
95 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
96 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
96 self.xmin = self.xmin if self.xmin else -self.xmax
97 self.xmin = self.xmin if self.xmin else -self.xmax
97 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
98 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
98 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
99 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
99 ax.plt = ax.pcolormesh(x, y, z[n].T,
100 ax.plt = ax.pcolormesh(x, y, z[n].T,
100 vmin=self.zmin,
101 vmin=self.zmin,
101 vmax=self.zmax,
102 vmax=self.zmax,
102 cmap=plt.get_cmap(self.colormap)
103 cmap=plt.get_cmap(self.colormap)
103 )
104 )
104
105
105 if self.showprofile:
106 if self.showprofile:
106 ax.plt_profile = self.pf_axes[n].plot(
107 ax.plt_profile = self.pf_axes[n].plot(
107 data['rti'][n], y)[0]
108 data['rti'][n], y)[0]
108 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
109 # ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
109 color="k", linestyle="dashed", lw=1)[0]
110 # color="k", linestyle="dashed", lw=1)[0]
110 if self.CODE == 'spc_moments':
111 if self.CODE == 'spc_moments':
111 ax.plt_mean = ax.plot(mean, y, color='k')[0]
112 ax.plt_mean = ax.plot(mean, y, color='k')[0]
112 else:
113 else:
113 ax.plt.set_array(z[n].T.ravel())
114 ax.plt.set_array(z[n].T.ravel())
114 if self.showprofile:
115 if self.showprofile:
115 ax.plt_profile.set_data(data['rti'][n], y)
116 ax.plt_profile.set_data(data['rti'][n], y)
116 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
117 #ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
117 if self.CODE == 'spc_moments':
118 if self.CODE == 'spc_moments':
118 ax.plt_mean.set_data(mean, y)
119 ax.plt_mean.set_data(mean, y)
119 if len(self.azimuthList) > 0 and len(self.elevationList) > 0:
120 if len(self.azimuthList) > 0 and len(self.elevationList) > 0:
120 self.titles.append('CH {}: {:2.1f}elv {:2.1f}az {:3.2f}dB'.format(self.channelList[n], noise, self.elevationList[n], self.azimuthList[n]))
121 self.titles.append('CH {}: {:2.1f}elv {:2.1f}az {:3.2f}dB'.format(self.channelList[n], noise, self.elevationList[n], self.azimuthList[n]))
121 else:
122 else:
122 self.titles.append('CH {}: {:3.2f}dB'.format(self.channelList[n], noise))
123 self.titles.append('CH {}: {:3.2f}dB'.format(self.channelList[n], noise))
123
124
124
125
125 class CrossSpectraPlot(Plot):
126 class CrossSpectraPlot(Plot):
126
127
127 CODE = 'cspc'
128 CODE = 'cspc'
128 colormap = 'jet'
129 colormap = 'jet'
129 plot_type = 'pcolor'
130 plot_type = 'pcolor'
130 zmin_coh = None
131 zmin_coh = None
131 zmax_coh = None
132 zmax_coh = None
132 zmin_phase = None
133 zmin_phase = None
133 zmax_phase = None
134 zmax_phase = None
134 realChannels = None
135 realChannels = None
135 crossPairs = None
136 crossPairs = None
136
137
137 def setup(self):
138 def setup(self):
138
139
139 self.ncols = 4
140 self.ncols = 4
140 self.nplots = len(self.data.pairs) * 2
141 self.nplots = len(self.data.pairs) * 2
141 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
142 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
142 self.width = 3.1 * self.ncols
143 self.width = 3.1 * self.ncols
143 self.height = 2.6 * self.nrows
144 self.height = 2.6 * self.nrows
144 self.ylabel = 'Range [km]'
145 self.ylabel = 'Range [km]'
145 self.showprofile = False
146 self.showprofile = False
146 self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
147 self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
147
148
148 def update(self, dataOut):
149 def update(self, dataOut):
149
150
150 data = {}
151 data = {}
151 meta = {}
152 meta = {}
152
153
153 spc = dataOut.data_spc
154 spc = dataOut.data_spc
154 cspc = dataOut.data_cspc
155 cspc = dataOut.data_cspc
155 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
156 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
156 rawPairs = list(combinations(list(range(dataOut.nChannels)), 2))
157 rawPairs = list(combinations(list(range(dataOut.nChannels)), 2))
157 meta['pairs'] = rawPairs
158 meta['pairs'] = rawPairs
158
159
159 if self.crossPairs == None:
160 if self.crossPairs == None:
160 self.crossPairs = dataOut.pairsList
161 self.crossPairs = dataOut.pairsList
161
162
162 tmp = []
163 tmp = []
163
164
164 for n, pair in enumerate(meta['pairs']):
165 for n, pair in enumerate(meta['pairs']):
165
166
166 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
167 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
167 coh = numpy.abs(out)
168 coh = numpy.abs(out)
168 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
169 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
169 tmp.append(coh)
170 tmp.append(coh)
170 tmp.append(phase)
171 tmp.append(phase)
171
172
172 data['cspc'] = numpy.array(tmp)
173 data['cspc'] = numpy.array(tmp)
173
174
174 return data, meta
175 return data, meta
175
176
176 def plot(self):
177 def plot(self):
177
178
178 if self.xaxis == "frequency":
179 if self.xaxis == "frequency":
179 x = self.data.xrange[0]
180 x = self.data.xrange[0]
180 self.xlabel = "Frequency (kHz)"
181 self.xlabel = "Frequency (kHz)"
181 elif self.xaxis == "time":
182 elif self.xaxis == "time":
182 x = self.data.xrange[1]
183 x = self.data.xrange[1]
183 self.xlabel = "Time (ms)"
184 self.xlabel = "Time (ms)"
184 else:
185 else:
185 x = self.data.xrange[2]
186 x = self.data.xrange[2]
186 self.xlabel = "Velocity (m/s)"
187 self.xlabel = "Velocity (m/s)"
187
188
188 self.titles = []
189 self.titles = []
189
190
190 y = self.data.yrange
191 y = self.data.yrange
191 self.y = y
192 self.y = y
192
193
193 data = self.data[-1]
194 data = self.data[-1]
194 cspc = data['cspc']
195 cspc = data['cspc']
195
196
196 for n in range(len(self.data.pairs)):
197 for n in range(len(self.data.pairs)):
197
198
198 pair = self.crossPairs[n]
199 pair = self.crossPairs[n]
199
200
200 coh = cspc[n*2]
201 coh = cspc[n*2]
201 phase = cspc[n*2+1]
202 phase = cspc[n*2+1]
202 ax = self.axes[2 * n]
203 ax = self.axes[2 * n]
203
204
204 if ax.firsttime:
205 if ax.firsttime:
205 ax.plt = ax.pcolormesh(x, y, coh.T,
206 ax.plt = ax.pcolormesh(x, y, coh.T,
206 vmin=self.zmin_coh,
207 vmin=self.zmin_coh,
207 vmax=self.zmax_coh,
208 vmax=self.zmax_coh,
208 cmap=plt.get_cmap(self.colormap_coh)
209 cmap=plt.get_cmap(self.colormap_coh)
209 )
210 )
210 else:
211 else:
211 ax.plt.set_array(coh.T.ravel())
212 ax.plt.set_array(coh.T.ravel())
212 self.titles.append(
213 self.titles.append(
213 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
214 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
214
215
215 ax = self.axes[2 * n + 1]
216 ax = self.axes[2 * n + 1]
216 if ax.firsttime:
217 if ax.firsttime:
217 ax.plt = ax.pcolormesh(x, y, phase.T,
218 ax.plt = ax.pcolormesh(x, y, phase.T,
218 vmin=-180,
219 vmin=-180,
219 vmax=180,
220 vmax=180,
220 cmap=plt.get_cmap(self.colormap_phase)
221 cmap=plt.get_cmap(self.colormap_phase)
221 )
222 )
222 else:
223 else:
223 ax.plt.set_array(phase.T.ravel())
224 ax.plt.set_array(phase.T.ravel())
224
225
225 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
226 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
226
227
227
228
228 class RTIPlot(Plot):
229 class RTIPlot(Plot):
229 '''
230 '''
230 Plot for RTI data
231 Plot for RTI data
231 '''
232 '''
232
233
233 CODE = 'rti'
234 CODE = 'rti'
234 colormap = 'jet'
235 colormap = 'jet'
235 plot_type = 'pcolorbuffer'
236 plot_type = 'pcolorbuffer'
236 titles = None
237 titles = None
237 channelList = []
238 channelList = []
238 elevationList = []
239 elevationList = []
239 azimuthList = []
240 azimuthList = []
240
241
241 def setup(self):
242 def setup(self):
242 self.xaxis = 'time'
243 self.xaxis = 'time'
243 self.ncols = 1
244 self.ncols = 1
244 #print("dataChannels ",self.data.channels)
245 #print("dataChannels ",self.data.channels)
245 self.nrows = len(self.data.channels)
246 self.nrows = len(self.data.channels)
246 self.nplots = len(self.data.channels)
247 self.nplots = len(self.data.channels)
247 self.ylabel = 'Range [km]'
248 self.ylabel = 'Range [km]'
248 self.xlabel = 'Time'
249 self.xlabel = 'Time'
249 self.cb_label = 'dB'
250 self.cb_label = 'dB'
250 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
251 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
251 self.titles = ['{} Channel {}'.format(
252 self.titles = ['{} Channel {}'.format(
252 self.CODE.upper(), x) for x in range(self.nplots)]
253 self.CODE.upper(), x) for x in range(self.nplots)]
253
254
254 def update_list(self,dataOut):
255 def update_list(self,dataOut):
255
256
256 if len(self.channelList) == 0:
257 if len(self.channelList) == 0:
257 self.channelList = dataOut.channelList
258 self.channelList = dataOut.channelList
258 if len(self.elevationList) == 0:
259 if len(self.elevationList) == 0:
259 self.elevationList = dataOut.elevationList
260 self.elevationList = dataOut.elevationList
260 if len(self.azimuthList) == 0:
261 if len(self.azimuthList) == 0:
261 self.azimuthList = dataOut.azimuthList
262 self.azimuthList = dataOut.azimuthList
262
263
263
264
264 def update(self, dataOut):
265 def update(self, dataOut):
265 if len(self.channelList) == 0:
266 if len(self.channelList) == 0:
266 self.update_list(dataOut)
267 self.update_list(dataOut)
267 data = {}
268 data = {}
268 meta = {}
269 meta = {}
269 data['rti'] = dataOut.getPower()
270 data['rti'] = dataOut.getPower()
270 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
271 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
271 return data, meta
272 return data, meta
272
273
273 def plot(self):
274 def plot(self):
274
275
275 self.x = self.data.times
276 self.x = self.data.times
276 self.y = self.data.yrange
277 self.y = self.data.yrange
277
278 #print(" x, y: ",self.x, self.y)
278 self.z = self.data[self.CODE]
279 self.z = self.data[self.CODE]
279 self.z = numpy.array(self.z, dtype=float)
280 self.z = numpy.array(self.z, dtype=float)
280 self.z = numpy.ma.masked_invalid(self.z)
281 self.z = numpy.ma.masked_invalid(self.z)
281
282
282 try:
283 try:
283 if self.channelList != None:
284 if self.channelList != None:
284 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
285 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
285 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
286 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
286 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
287 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
287 else:
288 else:
288 self.titles = ['{} Channel {}'.format(
289 self.titles = ['{} Channel {}'.format(
289 self.CODE.upper(), x) for x in self.channelList]
290 self.CODE.upper(), x) for x in self.channelList]
290 except:
291 except:
291 if self.channelList.any() != None:
292 if self.channelList.any() != None:
292
293
293 self.titles = ['{} Channel {}'.format(
294 self.titles = ['{} Channel {}'.format(
294 self.CODE.upper(), x) for x in self.channelList]
295 self.CODE.upper(), x) for x in self.channelList]
295
296
296 if self.decimation is None:
297 if self.decimation is None:
297 x, y, z = self.fill_gaps(self.x, self.y, self.z)
298 x, y, z = self.fill_gaps(self.x, self.y, self.z)
298 else:
299 else:
299 x, y, z = self.fill_gaps(*self.decimate())
300 x, y, z = self.fill_gaps(*self.decimate())
300
301
301 #dummy_var = self.axes #Extrañamente esto actualiza el valor axes
302 #dummy_var = self.axes #Extrañamente esto actualiza el valor axes
302 for n, ax in enumerate(self.axes):
303 for n, ax in enumerate(self.axes):
303 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
304 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
304 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
305 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
305 data = self.data[-1]
306 data = self.data[-1]
306
307
307 if ax.firsttime:
308 if ax.firsttime:
308 ax.plt = ax.pcolormesh(x, y, z[n].T,
309 ax.plt = ax.pcolormesh(x, y, z[n].T,
309 vmin=self.zmin,
310 vmin=self.zmin,
310 vmax=self.zmax,
311 vmax=self.zmax,
311 cmap=plt.get_cmap(self.colormap)
312 cmap=plt.get_cmap(self.colormap)
312 )
313 )
313 if self.showprofile:
314 if self.showprofile:
314 ax.plot_profile = self.pf_axes[n].plot(data[self.CODE][n], self.y)[0]
315 ax.plot_profile = self.pf_axes[n].plot(data[self.CODE][n], self.y)[0]
315 if "noise" in self.data:
316 if "noise" in self.data:
316
317
317 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(data['noise'][n], len(self.y)), self.y,
318 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(data['noise'][n], len(self.y)), self.y,
318 color="k", linestyle="dashed", lw=1)[0]
319 color="k", linestyle="dashed", lw=1)[0]
319 else:
320 else:
320 ax.collections.remove(ax.collections[0])
321 ax.collections.remove(ax.collections[0])
321 ax.plt = ax.pcolormesh(x, y, z[n].T,
322 ax.plt = ax.pcolormesh(x, y, z[n].T,
322 vmin=self.zmin,
323 vmin=self.zmin,
323 vmax=self.zmax,
324 vmax=self.zmax,
324 cmap=plt.get_cmap(self.colormap)
325 cmap=plt.get_cmap(self.colormap)
325 )
326 )
326 if self.showprofile:
327 if self.showprofile:
327 ax.plot_profile.set_data(data[self.CODE][n], self.y)
328 ax.plot_profile.set_data(data[self.CODE][n], self.y)
328 if "noise" in self.data:
329 if "noise" in self.data:
329
330
330 ax.plot_noise.set_data(numpy.repeat(
331 ax.plot_noise.set_data(numpy.repeat(
331 data['noise'][n], len(self.y)), self.y)
332 data['noise'][n], len(self.y)), self.y)
332
333
333
334
334 class CoherencePlot(RTIPlot):
335 class CoherencePlot(RTIPlot):
335 '''
336 '''
336 Plot for Coherence data
337 Plot for Coherence data
337 '''
338 '''
338
339
339 CODE = 'coh'
340 CODE = 'coh'
340
341
341 def setup(self):
342 def setup(self):
342 self.xaxis = 'time'
343 self.xaxis = 'time'
343 self.ncols = 1
344 self.ncols = 1
344 self.nrows = len(self.data.pairs)
345 self.nrows = len(self.data.pairs)
345 self.nplots = len(self.data.pairs)
346 self.nplots = len(self.data.pairs)
346 self.ylabel = 'Range [km]'
347 self.ylabel = 'Range [km]'
347 self.xlabel = 'Time'
348 self.xlabel = 'Time'
348 self.plots_adjust.update({'hspace':0.6, 'left': 0.1, 'bottom': 0.1,'right':0.95})
349 self.plots_adjust.update({'hspace':0.6, 'left': 0.1, 'bottom': 0.1,'right':0.95})
349 if self.CODE == 'coh':
350 if self.CODE == 'coh':
350 self.cb_label = ''
351 self.cb_label = ''
351 self.titles = [
352 self.titles = [
352 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
353 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
353 else:
354 else:
354 self.cb_label = 'Degrees'
355 self.cb_label = 'Degrees'
355 self.titles = [
356 self.titles = [
356 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
357 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
357
358
358 def update(self, dataOut):
359 def update(self, dataOut):
359 self.update_list(dataOut)
360 self.update_list(dataOut)
360 data = {}
361 data = {}
361 meta = {}
362 meta = {}
362 data['coh'] = dataOut.getCoherence()
363 data['coh'] = dataOut.getCoherence()
363 meta['pairs'] = dataOut.pairsList
364 meta['pairs'] = dataOut.pairsList
364
365
365
366
366 return data, meta
367 return data, meta
367
368
368 class PhasePlot(CoherencePlot):
369 class PhasePlot(CoherencePlot):
369 '''
370 '''
370 Plot for Phase map data
371 Plot for Phase map data
371 '''
372 '''
372
373
373 CODE = 'phase'
374 CODE = 'phase'
374 colormap = 'seismic'
375 colormap = 'seismic'
375
376
376 def update(self, dataOut):
377 def update(self, dataOut):
377
378
378 data = {}
379 data = {}
379 meta = {}
380 meta = {}
380 data['phase'] = dataOut.getCoherence(phase=True)
381 data['phase'] = dataOut.getCoherence(phase=True)
381 meta['pairs'] = dataOut.pairsList
382 meta['pairs'] = dataOut.pairsList
382
383
383 return data, meta
384 return data, meta
384
385
385 class NoisePlot(Plot):
386 class NoisePlot(Plot):
386 '''
387 '''
387 Plot for noise
388 Plot for noise
388 '''
389 '''
389
390
390 CODE = 'noise'
391 CODE = 'noise'
391 plot_type = 'scatterbuffer'
392 plot_type = 'scatterbuffer'
392
393
393 def setup(self):
394 def setup(self):
394 self.xaxis = 'time'
395 self.xaxis = 'time'
395 self.ncols = 1
396 self.ncols = 1
396 self.nrows = 1
397 self.nrows = 1
397 self.nplots = 1
398 self.nplots = 1
398 self.ylabel = 'Intensity [dB]'
399 self.ylabel = 'Intensity [dB]'
399 self.xlabel = 'Time'
400 self.xlabel = 'Time'
400 self.titles = ['Noise']
401 self.titles = ['Noise']
401 self.colorbar = False
402 self.colorbar = False
402 self.plots_adjust.update({'right': 0.85 })
403 self.plots_adjust.update({'right': 0.85 })
403
404
404 def update(self, dataOut):
405 def update(self, dataOut):
405
406
406 data = {}
407 data = {}
407 meta = {}
408 meta = {}
408 noise = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor).reshape(dataOut.nChannels, 1)
409 noise = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor).reshape(dataOut.nChannels, 1)
409 data['noise'] = noise
410 data['noise'] = noise
410 meta['yrange'] = numpy.array([])
411 meta['yrange'] = numpy.array([])
411
412
412 return data, meta
413 return data, meta
413
414
414 def plot(self):
415 def plot(self):
415
416
416 x = self.data.times
417 x = self.data.times
417 xmin = self.data.min_time
418 xmin = self.data.min_time
418 xmax = xmin + self.xrange * 60 * 60
419 xmax = xmin + self.xrange * 60 * 60
419 Y = self.data['noise']
420 Y = self.data['noise']
420
421
421 if self.axes[0].firsttime:
422 if self.axes[0].firsttime:
422 if self.ymin is None: self.ymin = numpy.nanmin(Y) - 5
423 if self.ymin is None: self.ymin = numpy.nanmin(Y) - 5
423 if self.ymax is None: self.ymax = numpy.nanmax(Y) + 5
424 if self.ymax is None: self.ymax = numpy.nanmax(Y) + 5
424 for ch in self.data.channels:
425 for ch in self.data.channels:
425 y = Y[ch]
426 y = Y[ch]
426 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
427 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
427 plt.legend(bbox_to_anchor=(1.18, 1.0))
428 plt.legend(bbox_to_anchor=(1.18, 1.0))
428 else:
429 else:
429 for ch in self.data.channels:
430 for ch in self.data.channels:
430 y = Y[ch]
431 y = Y[ch]
431 self.axes[0].lines[ch].set_data(x, y)
432 self.axes[0].lines[ch].set_data(x, y)
432
433
433
434
434 class PowerProfilePlot(Plot):
435 class PowerProfilePlot(Plot):
435
436
436 CODE = 'pow_profile'
437 CODE = 'pow_profile'
437 plot_type = 'scatter'
438 plot_type = 'scatter'
438
439
439 def setup(self):
440 def setup(self):
440
441
441 self.ncols = 1
442 self.ncols = 1
442 self.nrows = 1
443 self.nrows = 1
443 self.nplots = 1
444 self.nplots = 1
444 self.height = 4
445 self.height = 4
445 self.width = 3
446 self.width = 3
446 self.ylabel = 'Range [km]'
447 self.ylabel = 'Range [km]'
447 self.xlabel = 'Intensity [dB]'
448 self.xlabel = 'Intensity [dB]'
448 self.titles = ['Power Profile']
449 self.titles = ['Power Profile']
449 self.colorbar = False
450 self.colorbar = False
450
451
451 def update(self, dataOut):
452 def update(self, dataOut):
452
453
453 data = {}
454 data = {}
454 meta = {}
455 meta = {}
455 data[self.CODE] = dataOut.getPower()
456 data[self.CODE] = dataOut.getPower()
456
457
457 return data, meta
458 return data, meta
458
459
459 def plot(self):
460 def plot(self):
460
461
461 y = self.data.yrange
462 y = self.data.yrange
462 self.y = y
463 self.y = y
463
464
464 x = self.data[-1][self.CODE]
465 x = self.data[-1][self.CODE]
465
466
466 if self.xmin is None: self.xmin = numpy.nanmin(x)*0.9
467 if self.xmin is None: self.xmin = numpy.nanmin(x)*0.9
467 if self.xmax is None: self.xmax = numpy.nanmax(x)*1.1
468 if self.xmax is None: self.xmax = numpy.nanmax(x)*1.1
468
469
469 if self.axes[0].firsttime:
470 if self.axes[0].firsttime:
470 for ch in self.data.channels:
471 for ch in self.data.channels:
471 self.axes[0].plot(x[ch], y, lw=1, label='Ch{}'.format(ch))
472 self.axes[0].plot(x[ch], y, lw=1, label='Ch{}'.format(ch))
472 plt.legend()
473 plt.legend()
473 else:
474 else:
474 for ch in self.data.channels:
475 for ch in self.data.channels:
475 self.axes[0].lines[ch].set_data(x[ch], y)
476 self.axes[0].lines[ch].set_data(x[ch], y)
476
477
477
478
478 class SpectraCutPlot(Plot):
479 class SpectraCutPlot(Plot):
479
480
480 CODE = 'spc_cut'
481 CODE = 'spc_cut'
481 plot_type = 'scatter'
482 plot_type = 'scatter'
482 buffering = False
483 buffering = False
483 heights = []
484 heights = []
484 channelList = []
485 channelList = []
485 maintitle = "Spectra Cuts"
486 maintitle = "Spectra Cuts"
486 flag_setIndex = False
487 flag_setIndex = False
487
488
488 def setup(self):
489 def setup(self):
489
490
490 self.nplots = len(self.data.channels)
491 self.nplots = len(self.data.channels)
491 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
492 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
492 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
493 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
493 self.width = 4.2 * self.ncols + 2.5
494 self.width = 4.2 * self.ncols + 2.5
494 self.height = 4.8 * self.nrows
495 self.height = 4.8 * self.nrows
495 self.ylabel = 'Power [dB]'
496 self.ylabel = 'Power [dB]'
496 self.colorbar = False
497 self.colorbar = False
497 self.plots_adjust.update({'left':0.05, 'hspace':0.3, 'right': 0.9, 'bottom':0.08})
498 self.plots_adjust.update({'left':0.05, 'hspace':0.3, 'right': 0.9, 'bottom':0.08})
498
499
499 if len(self.selectedHeightsList) > 0:
500 if len(self.selectedHeightsList) > 0:
500 self.maintitle = "Spectra Cut"# for %d km " %(int(self.selectedHeight))
501 self.maintitle = "Spectra Cut"# for %d km " %(int(self.selectedHeight))
501
502
502
503
503
504
504 def update(self, dataOut):
505 def update(self, dataOut):
505 if len(self.channelList) == 0:
506 if len(self.channelList) == 0:
506 self.channelList = dataOut.channelList
507 self.channelList = dataOut.channelList
507
508
508 self.heights = dataOut.heightList
509 self.heights = dataOut.heightList
509 #print("sels: ",self.selectedHeightsList)
510 #print("sels: ",self.selectedHeightsList)
510 if len(self.selectedHeightsList)>0 and not self.flag_setIndex:
511 if len(self.selectedHeightsList)>0 and not self.flag_setIndex:
511
512
512 for sel_height in self.selectedHeightsList:
513 for sel_height in self.selectedHeightsList:
513 index_list = numpy.where(self.heights >= sel_height)
514 index_list = numpy.where(self.heights >= sel_height)
514 index_list = index_list[0]
515 index_list = index_list[0]
515 self.height_index.append(index_list[0])
516 self.height_index.append(index_list[0])
516 #print("sels i:"", self.height_index)
517 #print("sels i:"", self.height_index)
517 self.flag_setIndex = True
518 self.flag_setIndex = True
518 #print(self.height_index)
519 #print(self.height_index)
519 data = {}
520 data = {}
520 meta = {}
521 meta = {}
521 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
522 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
522
523
523 data['spc'] = spc
524 data['spc'] = spc
524 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
525 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
525
526
526 return data, meta
527 return data, meta
527
528
528 def plot(self):
529 def plot(self):
529 if self.xaxis == "frequency":
530 if self.xaxis == "frequency":
530 x = self.data.xrange[0][1:]
531 x = self.data.xrange[0][1:]
531 self.xlabel = "Frequency (kHz)"
532 self.xlabel = "Frequency (kHz)"
532 elif self.xaxis == "time":
533 elif self.xaxis == "time":
533 x = self.data.xrange[1]
534 x = self.data.xrange[1]
534 self.xlabel = "Time (ms)"
535 self.xlabel = "Time (ms)"
535 else:
536 else:
536 x = self.data.xrange[2]
537 x = self.data.xrange[2]
537 self.xlabel = "Velocity (m/s)"
538 self.xlabel = "Velocity (m/s)"
538
539
539 self.titles = []
540 self.titles = []
540
541
541 y = self.data.yrange
542 y = self.data.yrange
542 z = self.data[-1]['spc']
543 z = self.data[-1]['spc']
543 #print(z.shape)
544 #print(z.shape)
544 if len(self.height_index) > 0:
545 if len(self.height_index) > 0:
545 index = self.height_index
546 index = self.height_index
546 else:
547 else:
547 index = numpy.arange(0, len(y), int((len(y))/9))
548 index = numpy.arange(0, len(y), int((len(y))/9))
548 #print("inde x ", index, self.axes)
549 #print("inde x ", index, self.axes)
549
550
550 for n, ax in enumerate(self.axes):
551 for n, ax in enumerate(self.axes):
551
552
552 if ax.firsttime:
553 if ax.firsttime:
553
554
554
555
555 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
556 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
556 self.xmin = self.xmin if self.xmin else -self.xmax
557 self.xmin = self.xmin if self.xmin else -self.xmax
557 self.ymin = self.ymin if self.ymin else numpy.nanmin(z)
558 self.ymin = self.ymin if self.ymin else numpy.nanmin(z)
558 self.ymax = self.ymax if self.ymax else numpy.nanmax(z)
559 self.ymax = self.ymax if self.ymax else numpy.nanmax(z)
559
560
560
561
561 ax.plt = ax.plot(x, z[n, :, index].T)
562 ax.plt = ax.plot(x, z[n, :, index].T)
562 labels = ['Range = {:2.1f}km'.format(y[i]) for i in index]
563 labels = ['Range = {:2.1f}km'.format(y[i]) for i in index]
563 self.figures[0].legend(ax.plt, labels, loc='center right', prop={'size': 8})
564 self.figures[0].legend(ax.plt, labels, loc='center right', prop={'size': 8})
564 ax.minorticks_on()
565 ax.minorticks_on()
565 ax.grid(which='major', axis='both')
566 ax.grid(which='major', axis='both')
566 ax.grid(which='minor', axis='x')
567 ax.grid(which='minor', axis='x')
567 else:
568 else:
568 for i, line in enumerate(ax.plt):
569 for i, line in enumerate(ax.plt):
569 line.set_data(x, z[n, :, index[i]])
570 line.set_data(x, z[n, :, index[i]])
570
571
571
572
572 self.titles.append('CH {}'.format(self.channelList[n]))
573 self.titles.append('CH {}'.format(self.channelList[n]))
573 plt.suptitle(self.maintitle, fontsize=10)
574 plt.suptitle(self.maintitle, fontsize=10)
574
575
575
576
576 class BeaconPhase(Plot):
577 class BeaconPhase(Plot):
577
578
578 __isConfig = None
579 __isConfig = None
579 __nsubplots = None
580 __nsubplots = None
580
581
581 PREFIX = 'beacon_phase'
582 PREFIX = 'beacon_phase'
582
583
583 def __init__(self):
584 def __init__(self):
584 Plot.__init__(self)
585 Plot.__init__(self)
585 self.timerange = 24*60*60
586 self.timerange = 24*60*60
586 self.isConfig = False
587 self.isConfig = False
587 self.__nsubplots = 1
588 self.__nsubplots = 1
588 self.counter_imagwr = 0
589 self.counter_imagwr = 0
589 self.WIDTH = 800
590 self.WIDTH = 800
590 self.HEIGHT = 400
591 self.HEIGHT = 400
591 self.WIDTHPROF = 120
592 self.WIDTHPROF = 120
592 self.HEIGHTPROF = 0
593 self.HEIGHTPROF = 0
593 self.xdata = None
594 self.xdata = None
594 self.ydata = None
595 self.ydata = None
595
596
596 self.PLOT_CODE = BEACON_CODE
597 self.PLOT_CODE = BEACON_CODE
597
598
598 self.FTP_WEI = None
599 self.FTP_WEI = None
599 self.EXP_CODE = None
600 self.EXP_CODE = None
600 self.SUB_EXP_CODE = None
601 self.SUB_EXP_CODE = None
601 self.PLOT_POS = None
602 self.PLOT_POS = None
602
603
603 self.filename_phase = None
604 self.filename_phase = None
604
605
605 self.figfile = None
606 self.figfile = None
606
607
607 self.xmin = None
608 self.xmin = None
608 self.xmax = None
609 self.xmax = None
609
610
610 def getSubplots(self):
611 def getSubplots(self):
611
612
612 ncol = 1
613 ncol = 1
613 nrow = 1
614 nrow = 1
614
615
615 return nrow, ncol
616 return nrow, ncol
616
617
617 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
618 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
618
619
619 self.__showprofile = showprofile
620 self.__showprofile = showprofile
620 self.nplots = nplots
621 self.nplots = nplots
621
622
622 ncolspan = 7
623 ncolspan = 7
623 colspan = 6
624 colspan = 6
624 self.__nsubplots = 2
625 self.__nsubplots = 2
625
626
626 self.createFigure(id = id,
627 self.createFigure(id = id,
627 wintitle = wintitle,
628 wintitle = wintitle,
628 widthplot = self.WIDTH+self.WIDTHPROF,
629 widthplot = self.WIDTH+self.WIDTHPROF,
629 heightplot = self.HEIGHT+self.HEIGHTPROF,
630 heightplot = self.HEIGHT+self.HEIGHTPROF,
630 show=show)
631 show=show)
631
632
632 nrow, ncol = self.getSubplots()
633 nrow, ncol = self.getSubplots()
633
634
634 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
635 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
635
636
636 def save_phase(self, filename_phase):
637 def save_phase(self, filename_phase):
637 f = open(filename_phase,'w+')
638 f = open(filename_phase,'w+')
638 f.write('\n\n')
639 f.write('\n\n')
639 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
640 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
640 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
641 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
641 f.close()
642 f.close()
642
643
643 def save_data(self, filename_phase, data, data_datetime):
644 def save_data(self, filename_phase, data, data_datetime):
644 f=open(filename_phase,'a')
645 f=open(filename_phase,'a')
645 timetuple_data = data_datetime.timetuple()
646 timetuple_data = data_datetime.timetuple()
646 day = str(timetuple_data.tm_mday)
647 day = str(timetuple_data.tm_mday)
647 month = str(timetuple_data.tm_mon)
648 month = str(timetuple_data.tm_mon)
648 year = str(timetuple_data.tm_year)
649 year = str(timetuple_data.tm_year)
649 hour = str(timetuple_data.tm_hour)
650 hour = str(timetuple_data.tm_hour)
650 minute = str(timetuple_data.tm_min)
651 minute = str(timetuple_data.tm_min)
651 second = str(timetuple_data.tm_sec)
652 second = str(timetuple_data.tm_sec)
652 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
653 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
653 f.close()
654 f.close()
654
655
655 def plot(self):
656 def plot(self):
656 log.warning('TODO: Not yet implemented...')
657 log.warning('TODO: Not yet implemented...')
657
658
658 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
659 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
659 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
660 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
660 timerange=None,
661 timerange=None,
661 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
662 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
662 server=None, folder=None, username=None, password=None,
663 server=None, folder=None, username=None, password=None,
663 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
664 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
664
665
665 if dataOut.flagNoData:
666 if dataOut.flagNoData:
666 return dataOut
667 return dataOut
667
668
668 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
669 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
669 return
670 return
670
671
671 if pairsList == None:
672 if pairsList == None:
672 pairsIndexList = dataOut.pairsIndexList[:10]
673 pairsIndexList = dataOut.pairsIndexList[:10]
673 else:
674 else:
674 pairsIndexList = []
675 pairsIndexList = []
675 for pair in pairsList:
676 for pair in pairsList:
676 if pair not in dataOut.pairsList:
677 if pair not in dataOut.pairsList:
677 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
678 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
678 pairsIndexList.append(dataOut.pairsList.index(pair))
679 pairsIndexList.append(dataOut.pairsList.index(pair))
679
680
680 if pairsIndexList == []:
681 if pairsIndexList == []:
681 return
682 return
682
683
683 # if len(pairsIndexList) > 4:
684 # if len(pairsIndexList) > 4:
684 # pairsIndexList = pairsIndexList[0:4]
685 # pairsIndexList = pairsIndexList[0:4]
685
686
686 hmin_index = None
687 hmin_index = None
687 hmax_index = None
688 hmax_index = None
688
689
689 if hmin != None and hmax != None:
690 if hmin != None and hmax != None:
690 indexes = numpy.arange(dataOut.nHeights)
691 indexes = numpy.arange(dataOut.nHeights)
691 hmin_list = indexes[dataOut.heightList >= hmin]
692 hmin_list = indexes[dataOut.heightList >= hmin]
692 hmax_list = indexes[dataOut.heightList <= hmax]
693 hmax_list = indexes[dataOut.heightList <= hmax]
693
694
694 if hmin_list.any():
695 if hmin_list.any():
695 hmin_index = hmin_list[0]
696 hmin_index = hmin_list[0]
696
697
697 if hmax_list.any():
698 if hmax_list.any():
698 hmax_index = hmax_list[-1]+1
699 hmax_index = hmax_list[-1]+1
699
700
700 x = dataOut.getTimeRange()
701 x = dataOut.getTimeRange()
701
702
702 thisDatetime = dataOut.datatime
703 thisDatetime = dataOut.datatime
703
704
704 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
705 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
705 xlabel = "Local Time"
706 xlabel = "Local Time"
706 ylabel = "Phase (degrees)"
707 ylabel = "Phase (degrees)"
707
708
708 update_figfile = False
709 update_figfile = False
709
710
710 nplots = len(pairsIndexList)
711 nplots = len(pairsIndexList)
711 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
712 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
712 phase_beacon = numpy.zeros(len(pairsIndexList))
713 phase_beacon = numpy.zeros(len(pairsIndexList))
713 for i in range(nplots):
714 for i in range(nplots):
714 pair = dataOut.pairsList[pairsIndexList[i]]
715 pair = dataOut.pairsList[pairsIndexList[i]]
715 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
716 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
716 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
717 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
717 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
718 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
718 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
719 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
719 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
720 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
720
721
721 if dataOut.beacon_heiIndexList:
722 if dataOut.beacon_heiIndexList:
722 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
723 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
723 else:
724 else:
724 phase_beacon[i] = numpy.average(phase)
725 phase_beacon[i] = numpy.average(phase)
725
726
726 if not self.isConfig:
727 if not self.isConfig:
727
728
728 nplots = len(pairsIndexList)
729 nplots = len(pairsIndexList)
729
730
730 self.setup(id=id,
731 self.setup(id=id,
731 nplots=nplots,
732 nplots=nplots,
732 wintitle=wintitle,
733 wintitle=wintitle,
733 showprofile=showprofile,
734 showprofile=showprofile,
734 show=show)
735 show=show)
735
736
736 if timerange != None:
737 if timerange != None:
737 self.timerange = timerange
738 self.timerange = timerange
738
739
739 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
740 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
740
741
741 if ymin == None: ymin = 0
742 if ymin == None: ymin = 0
742 if ymax == None: ymax = 360
743 if ymax == None: ymax = 360
743
744
744 self.FTP_WEI = ftp_wei
745 self.FTP_WEI = ftp_wei
745 self.EXP_CODE = exp_code
746 self.EXP_CODE = exp_code
746 self.SUB_EXP_CODE = sub_exp_code
747 self.SUB_EXP_CODE = sub_exp_code
747 self.PLOT_POS = plot_pos
748 self.PLOT_POS = plot_pos
748
749
749 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
750 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
750 self.isConfig = True
751 self.isConfig = True
751 self.figfile = figfile
752 self.figfile = figfile
752 self.xdata = numpy.array([])
753 self.xdata = numpy.array([])
753 self.ydata = numpy.array([])
754 self.ydata = numpy.array([])
754
755
755 update_figfile = True
756 update_figfile = True
756
757
757 #open file beacon phase
758 #open file beacon phase
758 path = '%s%03d' %(self.PREFIX, self.id)
759 path = '%s%03d' %(self.PREFIX, self.id)
759 beacon_file = os.path.join(path,'%s.txt'%self.name)
760 beacon_file = os.path.join(path,'%s.txt'%self.name)
760 self.filename_phase = os.path.join(figpath,beacon_file)
761 self.filename_phase = os.path.join(figpath,beacon_file)
761 #self.save_phase(self.filename_phase)
762 #self.save_phase(self.filename_phase)
762
763
763
764
764 #store data beacon phase
765 #store data beacon phase
765 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
766 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
766
767
767 self.setWinTitle(title)
768 self.setWinTitle(title)
768
769
769
770
770 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
771 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
771
772
772 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
773 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
773
774
774 axes = self.axesList[0]
775 axes = self.axesList[0]
775
776
776 self.xdata = numpy.hstack((self.xdata, x[0:1]))
777 self.xdata = numpy.hstack((self.xdata, x[0:1]))
777
778
778 if len(self.ydata)==0:
779 if len(self.ydata)==0:
779 self.ydata = phase_beacon.reshape(-1,1)
780 self.ydata = phase_beacon.reshape(-1,1)
780 else:
781 else:
781 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
782 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
782
783
783
784
784 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
785 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
785 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
786 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
786 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
787 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
787 XAxisAsTime=True, grid='both'
788 XAxisAsTime=True, grid='both'
788 )
789 )
789
790
790 self.draw()
791 self.draw()
791
792
792 if dataOut.ltctime >= self.xmax:
793 if dataOut.ltctime >= self.xmax:
793 self.counter_imagwr = wr_period
794 self.counter_imagwr = wr_period
794 self.isConfig = False
795 self.isConfig = False
795 update_figfile = True
796 update_figfile = True
796
797
797 self.save(figpath=figpath,
798 self.save(figpath=figpath,
798 figfile=figfile,
799 figfile=figfile,
799 save=save,
800 save=save,
800 ftp=ftp,
801 ftp=ftp,
801 wr_period=wr_period,
802 wr_period=wr_period,
802 thisDatetime=thisDatetime,
803 thisDatetime=thisDatetime,
803 update_figfile=update_figfile)
804 update_figfile=update_figfile)
804
805
805 return dataOut
806 return dataOut
806
807
807 class NoiselessSpectraPlot(Plot):
808 class NoiselessSpectraPlot(Plot):
808 '''
809 '''
809 Plot for Spectra data, subtracting
810 Plot for Spectra data, subtracting
810 the noise in all channels, using for
811 the noise in all channels, using for
811 amisr-14 data
812 amisr-14 data
812 '''
813 '''
813
814
814 CODE = 'noiseless_spc'
815 CODE = 'noiseless_spc'
815 colormap = 'nipy_spectral'
816 colormap = 'nipy_spectral'
816 plot_type = 'pcolor'
817 plot_type = 'pcolor'
817 buffering = False
818 buffering = False
818 channelList = []
819 channelList = []
820 last_noise = None
819
821
820 def setup(self):
822 def setup(self):
821
823
822 self.nplots = len(self.data.channels)
824 self.nplots = len(self.data.channels)
823 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
825 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
824 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
826 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
825 self.height = 2.6 * self.nrows
827 self.height = 2.6 * self.nrows
826
828
827 self.cb_label = 'dB'
829 self.cb_label = 'dB'
828 if self.showprofile:
830 if self.showprofile:
829 self.width = 4 * self.ncols
831 self.width = 4 * self.ncols
830 else:
832 else:
831 self.width = 3.5 * self.ncols
833 self.width = 3.5 * self.ncols
832 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.08})
834 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.08})
833 self.ylabel = 'Range [km]'
835 self.ylabel = 'Range [km]'
834
836
835
837
836 def update_list(self,dataOut):
838 def update_list(self,dataOut):
837 if len(self.channelList) == 0:
839 if len(self.channelList) == 0:
838 self.channelList = dataOut.channelList
840 self.channelList = dataOut.channelList
839
841
840 def update(self, dataOut):
842 def update(self, dataOut):
841
843
842 self.update_list(dataOut)
844 self.update_list(dataOut)
843 data = {}
845 data = {}
844 meta = {}
846 meta = {}
845 n0 = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
846 (nch, nff, nh) = dataOut.data_spc.shape
847 n1 = numpy.repeat(n0,nh, axis=0).reshape((nch,nh))
848 noise = numpy.repeat(n1,nff, axis=1).reshape((nch,nff,nh))
849 #print(noise.shape, "noise", noise)
850
847
851 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor) - noise
848 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
849 n0 = 10*numpy.log10(dataOut.getNoise()/float(norm))
852
850
853 data['spc'] = spc
851 if self.last_noise == None:
854 data['rti'] = dataOut.getPower() - n1
852 self.last_noise = n0
853 else:
854 n0 = (n0*0.2 + self.last_noise*0.8)
855 self.last_noise = n0
856
857 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
858
859 data['spc'] = spc - n0
860 data['rti'] = dataOut.getPower() - n0
855
861
856 data['noise'] = n0
862 #data['noise'] = noise
857 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
863 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
858
864
859 return data, meta
865 return data, meta
860
866
861 def plot(self):
867 def plot(self):
862 if self.xaxis == "frequency":
868 if self.xaxis == "frequency":
863 x = self.data.xrange[0]
869 x = self.data.xrange[0]
864 self.xlabel = "Frequency (kHz)"
870 self.xlabel = "Frequency (kHz)"
865 elif self.xaxis == "time":
871 elif self.xaxis == "time":
866 x = self.data.xrange[1]
872 x = self.data.xrange[1]
867 self.xlabel = "Time (ms)"
873 self.xlabel = "Time (ms)"
868 else:
874 else:
869 x = self.data.xrange[2]
875 x = self.data.xrange[2]
870 self.xlabel = "Velocity (m/s)"
876 self.xlabel = "Velocity (m/s)"
871
877
872 self.titles = []
878 self.titles = []
873 y = self.data.yrange
879 y = self.data.yrange
874 self.y = y
880 self.y = y
875
881
876 data = self.data[-1]
882 data = self.data[-1]
877 z = data['spc']
883 z = data['spc']
878
884
879 for n, ax in enumerate(self.axes):
885 for n, ax in enumerate(self.axes):
880 noise = data['noise'][n]
886 #noise = data['noise'][n]
881
887
882 if ax.firsttime:
888 if ax.firsttime:
883 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
889 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
884 self.xmin = self.xmin if self.xmin else -self.xmax
890 self.xmin = self.xmin if self.xmin else -self.xmax
885 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
891 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
886 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
892 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
887 ax.plt = ax.pcolormesh(x, y, z[n].T,
893 ax.plt = ax.pcolormesh(x, y, z[n].T,
888 vmin=self.zmin,
894 vmin=self.zmin,
889 vmax=self.zmax,
895 vmax=self.zmax,
890 cmap=plt.get_cmap(self.colormap)
896 cmap=plt.get_cmap(self.colormap)
891 )
897 )
892
898
893 if self.showprofile:
899 if self.showprofile:
894 ax.plt_profile = self.pf_axes[n].plot(
900 ax.plt_profile = self.pf_axes[n].plot(
895 data['rti'][n], y)[0]
901 data['rti'][n], y)[0]
896 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
902
897 color="k", linestyle="dashed", lw=1)[0]
898
903
899 else:
904 else:
900 ax.plt.set_array(z[n].T.ravel())
905 ax.plt.set_array(z[n].T.ravel())
901 if self.showprofile:
906 if self.showprofile:
902 ax.plt_profile.set_data(data['rti'][n], y)
907 ax.plt_profile.set_data(data['rti'][n], y)
903 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
904
908
905 self.titles.append('CH {}: {:3.2f}dB'.format(self.channelList[n], noise))
909
910 self.titles.append('CH {}'.format(self.channelList[n]))
906
911
907
912
908 class NoiselessRTIPlot(Plot):
913 class NoiselessRTIPlot(Plot):
909 '''
914 '''
910 Plot for RTI data
915 Plot for RTI data
911 '''
916 '''
912
917
913 CODE = 'noiseless_rti'
918 CODE = 'noiseless_rti'
914 colormap = 'jet'
919 colormap = 'jet'
915 plot_type = 'pcolorbuffer'
920 plot_type = 'pcolorbuffer'
916 titles = None
921 titles = None
917 channelList = []
922 channelList = []
918 elevationList = []
923 elevationList = []
919 azimuthList = []
924 azimuthList = []
925 last_noise = None
920
926
921 def setup(self):
927 def setup(self):
922 self.xaxis = 'time'
928 self.xaxis = 'time'
923 self.ncols = 1
929 self.ncols = 1
924 #print("dataChannels ",self.data.channels)
930 #print("dataChannels ",self.data.channels)
925 self.nrows = len(self.data.channels)
931 self.nrows = len(self.data.channels)
926 self.nplots = len(self.data.channels)
932 self.nplots = len(self.data.channels)
927 self.ylabel = 'Range [km]'
933 self.ylabel = 'Range [km]'
928 self.xlabel = 'Time'
934 self.xlabel = 'Time'
929 self.cb_label = 'dB'
935 self.cb_label = 'dB'
930 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
936 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
931 self.titles = ['{} Channel {}'.format(
937 self.titles = ['{} Channel {}'.format(
932 self.CODE.upper(), x) for x in range(self.nplots)]
938 self.CODE.upper(), x) for x in range(self.nplots)]
933
939
934 def update_list(self,dataOut):
940 def update_list(self,dataOut):
935 if len(self.channelList) == 0:
941 if len(self.channelList) == 0:
936 self.channelList = dataOut.channelList
942 self.channelList = dataOut.channelList
937 if len(self.elevationList) == 0:
943 if len(self.elevationList) == 0:
938 self.elevationList = dataOut.elevationList
944 self.elevationList = dataOut.elevationList
939 if len(self.azimuthList) == 0:
945 if len(self.azimuthList) == 0:
940 self.azimuthList = dataOut.azimuthList
946 self.azimuthList = dataOut.azimuthList
941
947
942 def update(self, dataOut):
948 def update(self, dataOut):
943 if len(self.channelList) == 0:
949 if len(self.channelList) == 0:
944 self.update_list(dataOut)
950 self.update_list(dataOut)
945 data = {}
951 data = {}
946 meta = {}
952 meta = {}
947
953
948 n0 = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
954
949 (nch, nff, nh) = dataOut.data_spc.shape
955 norm = dataOut.nProfiles * dataOut.max_nIncohInt * dataOut.nCohInt * dataOut.windowOfFilter
950 #print(nch, nff, nh)
956 n0 = 10*numpy.log10(dataOut.getNoise()/float(norm))
951 if nch != 1:
957 #print("noise: ",n0, dataOut.normFactor, norm, dataOut.nIncohInt, dataOut.max_nIncohInt)
952 aux = []
958 if self.last_noise == None:
953 for c in self.channelList:
959 self.last_noise = n0
954 aux.append(n0[c])
960 else:
955 n0 = numpy.asarray(aux)
961 n0 = (n0*0.2 + self.last_noise*0.8)
956 noise = numpy.repeat(n0,nh, axis=0).reshape((nch,nh))
962 self.last_noise = n0
957 #print(dataOut.elevationList, dataOut.azimuthList)
963
958 #print(dataOut.channelList)
964
959 data['noiseless_rti'] = dataOut.getPower() - noise
965 data['noiseless_rti'] = dataOut.getPower() - n0
960 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
966
967 #data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
968 #print(noise)
961 return data, meta
969 return data, meta
962
970
963 def plot(self):
971 def plot(self):
964
972
965 self.x = self.data.times
973 self.x = self.data.times
966 self.y = self.data.yrange
974 self.y = self.data.yrange
967 self.z = self.data['noiseless_rti']
975 self.z = self.data['noiseless_rti']
968 self.z = numpy.array(self.z, dtype=float)
976 self.z = numpy.array(self.z, dtype=float)
969 self.z = numpy.ma.masked_invalid(self.z)
977 self.z = numpy.ma.masked_invalid(self.z)
970
978
971 try:
979 try:
972 if self.channelList != None:
980 if self.channelList != None:
973 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
981 if len(self.elevationList) > 0 and len(self.azimuthList) > 0:
974 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
982 self.titles = ['{} Channel {} ({:2.1f} Elev, {:2.1f} Azth)'.format(
975 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
983 self.CODE.upper(), x, self.elevationList[x], self.azimuthList[x]) for x in self.channelList]
976 else:
984 else:
977 self.titles = ['{} Channel {}'.format(
985 self.titles = ['{} Channel {}'.format(
978 self.CODE.upper(), x) for x in self.channelList]
986 self.CODE.upper(), x) for x in self.channelList]
979 except:
987 except:
980 if self.channelList.any() != None:
988 if self.channelList.any() != None:
981
989
982 self.titles = ['{} Channel {}'.format(
990 self.titles = ['{} Channel {}'.format(
983 self.CODE.upper(), x) for x in self.channelList]
991 self.CODE.upper(), x) for x in self.channelList]
984 if self.decimation is None:
992 if self.decimation is None:
985 x, y, z = self.fill_gaps(self.x, self.y, self.z)
993 x, y, z = self.fill_gaps(self.x, self.y, self.z)
986 else:
994 else:
987 x, y, z = self.fill_gaps(*self.decimate())
995 x, y, z = self.fill_gaps(*self.decimate())
988 dummy_var = self.axes #Extrañamente esto actualiza el valor axes
996 dummy_var = self.axes #Extrañamente esto actualiza el valor axes
997 #print("plot shapes ", z.shape, x.shape, y.shape)
989 for n, ax in enumerate(self.axes):
998 for n, ax in enumerate(self.axes):
990 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
999 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
991 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
1000 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
992 data = self.data[-1]
1001 data = self.data[-1]
993 if ax.firsttime:
1002 if ax.firsttime:
994 ax.plt = ax.pcolormesh(x, y, z[n].T,
1003 ax.plt = ax.pcolormesh(x, y, z[n].T,
995 vmin=self.zmin,
1004 vmin=self.zmin,
996 vmax=self.zmax,
1005 vmax=self.zmax,
997 cmap=plt.get_cmap(self.colormap)
1006 cmap=plt.get_cmap(self.colormap)
998 )
1007 )
999 if self.showprofile:
1008 if self.showprofile:
1000 ax.plot_profile = self.pf_axes[n].plot(data['noiseless_rti'][n], self.y)[0]
1009 ax.plot_profile = self.pf_axes[n].plot(data['noiseless_rti'][n], self.y)[0]
1001
1010
1002 if "noise" in self.data:
1003 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(data['noise'][n], len(self.y)), self.y,
1004 color="k", linestyle="dashed", lw=1)[0]
1005 else:
1011 else:
1006 ax.collections.remove(ax.collections[0])
1012 ax.collections.remove(ax.collections[0])
1007 ax.plt = ax.pcolormesh(x, y, z[n].T,
1013 ax.plt = ax.pcolormesh(x, y, z[n].T,
1008 vmin=self.zmin,
1014 vmin=self.zmin,
1009 vmax=self.zmax,
1015 vmax=self.zmax,
1010 cmap=plt.get_cmap(self.colormap)
1016 cmap=plt.get_cmap(self.colormap)
1011 )
1017 )
1012 if self.showprofile:
1018 if self.showprofile:
1013 ax.plot_profile.set_data(data['noiseless_rti'][n], self.y)
1019 ax.plot_profile.set_data(data['noiseless_rti'][n], self.y)
1014 if "noise" in self.data:
1020 # if "noise" in self.data:
1015 ax.plot_noise.set_data(numpy.repeat(
1021 # #ax.plot_noise.set_data(numpy.repeat(data['noise'][n], len(self.y)), self.y)
1016 data['noise'][n], len(self.y)), self.y)
1022 # ax.plot_noise.set_data(data['noise'][n], self.y)
1023
1024
1025 class OutliersRTIPlot(Plot):
1026 '''
1027 Plot for data_xxxx object
1028 '''
1029
1030 CODE = 'outlier'
1031 colormap = 'cool'
1032 plot_type = 'pcolorbuffer'
1033
1034 def setup(self):
1035 self.xaxis = 'time'
1036 self.ncols = 1
1037 self.nrows = self.data.shape('outlier')[0]
1038 self.nplots = self.nrows
1039 self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
1040
1041
1042 if not self.xlabel:
1043 self.xlabel = 'Time'
1044
1045 self.ylabel = 'Height [km]'
1046 if not self.titles:
1047 self.titles = ['Outliers Ch:{}'.format(x) for x in range(self.nrows)]
1048
1049 def update(self, dataOut):
1050
1051 data = {}
1052 data['outlier'] = dataOut.data_outlier
1053
1054 meta = {}
1055
1056 return data, meta
1057
1058 def plot(self):
1059 # self.data.normalize_heights()
1060 self.x = self.data.times
1061 self.y = self.data.yrange
1062 self.z = self.data['outlier']
1063
1064 #self.z = numpy.ma.masked_invalid(self.z)
1065
1066 if self.decimation is None:
1067 x, y, z = self.fill_gaps(self.x, self.y, self.z)
1068 else:
1069 x, y, z = self.fill_gaps(*self.decimate())
1070
1071 for n, ax in enumerate(self.axes):
1072
1073 self.zmax = self.zmax if self.zmax is not None else numpy.max(
1074 self.z[n])
1075 self.zmin = self.zmin if self.zmin is not None else numpy.min(
1076 self.z[n])
1077 data = self.data[-1]
1078 if ax.firsttime:
1079 if self.zlimits is not None:
1080 self.zmin, self.zmax = self.zlimits[n]
1081
1082 ax.plt = ax.pcolormesh(x, y, z[n].T,
1083 vmin=self.zmin,
1084 vmax=self.zmax,
1085 cmap=self.cmaps[n]
1086 )
1087 if self.showprofile:
1088 ax.plot_profile = self.pf_axes[n].plot(data['outlier'][n], self.y)[0]
1089 self.pf_axes[n].set_xlabel('')
1090 else:
1091 if self.zlimits is not None:
1092 self.zmin, self.zmax = self.zlimits[n]
1093 ax.collections.remove(ax.collections[0])
1094 ax.plt = ax.pcolormesh(x, y, z[n].T ,
1095 vmin=self.zmin,
1096 vmax=self.zmax,
1097 cmap=self.cmaps[n]
1098 )
1099 if self.showprofile:
1100 ax.plot_profile.set_data(data['outlier'][n], self.y)
1101 self.pf_axes[n].set_xlabel('')
1102
1103 # class NoiseRTIPlot(Plot):
1104 # '''
1105 # Plot for data_xxxx object
1106 # '''
1107 #
1108 # CODE = 'noise'
1109 # colormap = 'jet'
1110 # plot_type = 'pcolorbuffer'
1111 #
1112 # def setup(self):
1113 # self.xaxis = 'time'
1114 # self.ncols = 1
1115 # self.nrows = self.data.shape('noise')[0]
1116 # self.nplots = self.nrows
1117 # self.plots_adjust.update({'hspace':0.8, 'left': 0.08, 'bottom': 0.2, 'right':0.94})
1118 #
1119 #
1120 # if not self.xlabel:
1121 # self.xlabel = 'Time'
1122 #
1123 # self.ylabel = 'Height [km]'
1124 # if not self.titles:
1125 # self.titles = ['noise Ch:{}'.format(x) for x in range(self.nrows)]
1126 #
1127 # def update(self, dataOut):
1128 #
1129 # data = {}
1130 # norm = dataOut.max_nIncohInt*dataOut.nProfiles* dataOut.nCohInt*dataOut.windowOfFilter
1131 # print("max incoh: ",dataOut.max_nIncohInt )
1132 # n0 = 10*numpy.log10(dataOut.getNoise()/norm)
1133 # data['noise'] = n0
1134 #
1135 # meta = {}
1136 #
1137 # return data, meta
1138 #
1139 # def plot(self):
1140 # # self.data.normalize_heights()
1141 # self.x = self.data.times
1142 # self.y = self.data.yrange
1143 # self.z = self.data['noise']
1144 #
1145 # #self.z = numpy.ma.masked_invalid(self.z)
1146 #
1147 # if self.decimation is None:
1148 # x, y, z = self.fill_gaps(self.x, self.y, self.z)
1149 # else:
1150 # x, y, z = self.fill_gaps(*self.decimate())
1151 #
1152 # for n, ax in enumerate(self.axes):
1153 #
1154 # self.zmax = self.zmax if self.zmax is not None else numpy.max(
1155 # self.z[n])
1156 # self.zmin = self.zmin if self.zmin is not None else numpy.min(
1157 # self.z[n])
1158 # data = self.data[-1]
1159 # if ax.firsttime:
1160 # if self.zlimits is not None:
1161 # self.zmin, self.zmax = self.zlimits[n]
1162 #
1163 # ax.plt = ax.pcolormesh(x, y, z[n].T,
1164 # vmin=self.zmin,
1165 # vmax=self.zmax,
1166 # cmap=self.cmaps[n]
1167 # )
1168 # if self.showprofile:
1169 # ax.plot_profile = self.pf_axes[n].plot(data['noise'][n], self.y)[0]
1170 # else:
1171 # if self.zlimits is not None:
1172 # self.zmin, self.zmax = self.zlimits[n]
1173 # ax.collections.remove(ax.collections[0])
1174 # ax.plt = ax.pcolormesh(x, y, z[n].T ,
1175 # vmin=self.zmin,
1176 # vmax=self.zmax,
1177 # cmap=self.cmaps[n]
1178 # )
1179 # if self.showprofile:
1180 # ax.plot_profile.set_data(data['noise'][n], self.y)
Show file before | Show file after | Hide comments
@@ -1,665 +1,666
1 ''''
1 ''''
2 Created on Set 9, 2015
2 Created on Set 9, 2015
3
3
4 @author: roj-idl71 Karim Kuyeng
4 @author: roj-idl71 Karim Kuyeng
5
5
6 @update: 2021, Joab Apaza
6 @update: 2021, Joab Apaza
7 '''
7 '''
8
8
9 import os
9 import os
10 import sys
10 import sys
11 import glob
11 import glob
12 import fnmatch
12 import fnmatch
13 import datetime
13 import datetime
14 import time
14 import time
15 import re
15 import re
16 import h5py
16 import h5py
17 import numpy
17 import numpy
18
18
19 try:
19 try:
20 from gevent import sleep
20 from gevent import sleep
21 except:
21 except:
22 from time import sleep
22 from time import sleep
23
23
24 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
24 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
25 from schainpy.model.data.jrodata import Voltage
25 from schainpy.model.data.jrodata import Voltage
26 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
26 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
27 from numpy import imag
27 from numpy import imag
28 from schainpy.utils import log
28 from schainpy.utils import log
29
29
30
30
31 class AMISRReader(ProcessingUnit):
31 class AMISRReader(ProcessingUnit):
32 '''
32 '''
33 classdocs
33 classdocs
34 '''
34 '''
35
35
36 def __init__(self):
36 def __init__(self):
37 '''
37 '''
38 Constructor
38 Constructor
39 '''
39 '''
40
40
41 ProcessingUnit.__init__(self)
41 ProcessingUnit.__init__(self)
42
42
43 self.set = None
43 self.set = None
44 self.subset = None
44 self.subset = None
45 self.extension_file = '.h5'
45 self.extension_file = '.h5'
46 self.dtc_str = 'dtc'
46 self.dtc_str = 'dtc'
47 self.dtc_id = 0
47 self.dtc_id = 0
48 self.status = True
48 self.status = True
49 self.isConfig = False
49 self.isConfig = False
50 self.dirnameList = []
50 self.dirnameList = []
51 self.filenameList = []
51 self.filenameList = []
52 self.fileIndex = None
52 self.fileIndex = None
53 self.flagNoMoreFiles = False
53 self.flagNoMoreFiles = False
54 self.flagIsNewFile = 0
54 self.flagIsNewFile = 0
55 self.filename = ''
55 self.filename = ''
56 self.amisrFilePointer = None
56 self.amisrFilePointer = None
57 self.realBeamCode = []
57 self.realBeamCode = []
58 self.beamCodeMap = None
58 self.beamCodeMap = None
59 self.azimuthList = []
59 self.azimuthList = []
60 self.elevationList = []
60 self.elevationList = []
61 self.dataShape = None
61 self.dataShape = None
62 self.flag_old_beams = False
62 self.flag_old_beams = False
63
63
64
64
65 self.profileIndex = 0
65 self.profileIndex = 0
66
66
67
67
68 self.beamCodeByFrame = None
68 self.beamCodeByFrame = None
69 self.radacTimeByFrame = None
69 self.radacTimeByFrame = None
70
70
71 self.dataset = None
71 self.dataset = None
72
72
73 self.__firstFile = True
73 self.__firstFile = True
74
74
75 self.buffer = None
75 self.buffer = None
76
76
77 self.timezone = 'ut'
77 self.timezone = 'ut'
78
78
79 self.__waitForNewFile = 20
79 self.__waitForNewFile = 20
80 self.__filename_online = None
80 self.__filename_online = None
81 #Is really necessary create the output object in the initializer
81 #Is really necessary create the output object in the initializer
82 self.dataOut = Voltage()
82 self.dataOut = Voltage()
83 self.dataOut.error=False
83 self.dataOut.error=False
84 self.margin_days = 1
84 self.margin_days = 1
85
85
86 def setup(self,path=None,
86 def setup(self,path=None,
87 startDate=None,
87 startDate=None,
88 endDate=None,
88 endDate=None,
89 startTime=None,
89 startTime=None,
90 endTime=None,
90 endTime=None,
91 walk=True,
91 walk=True,
92 timezone='ut',
92 timezone='ut',
93 all=0,
93 all=0,
94 code = None,
94 code = None,
95 nCode = 0,
95 nCode = 0,
96 nBaud = 0,
96 nBaud = 0,
97 online=False,
97 online=False,
98 old_beams=False,
98 old_beams=False,
99 margin_days=1):
99 margin_days=1):
100
100
101
101
102
102
103 self.timezone = timezone
103 self.timezone = timezone
104 self.all = all
104 self.all = all
105 self.online = online
105 self.online = online
106 self.flag_old_beams = old_beams
106 self.flag_old_beams = old_beams
107 self.code = code
107 self.code = code
108 self.nCode = int(nCode)
108 self.nCode = int(nCode)
109 self.nBaud = int(nBaud)
109 self.nBaud = int(nBaud)
110 self.margin_days = margin_days
110 self.margin_days = margin_days
111
111
112
112
113 #self.findFiles()
113 #self.findFiles()
114 if not(online):
114 if not(online):
115 #Busqueda de archivos offline
115 #Busqueda de archivos offline
116 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
116 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
117 else:
117 else:
118 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
118 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
119
119
120 if not(self.filenameList):
120 if not(self.filenameList):
121 raise schainpy.admin.SchainWarning("There is no files into the folder: %s"%(path))
121 raise schainpy.admin.SchainWarning("There is no files into the folder: %s"%(path))
122 sys.exit()
122 sys.exit()
123
123
124 self.fileIndex = 0
124 self.fileIndex = 0
125
125
126 self.readNextFile(online)
126 self.readNextFile(online)
127
127
128 '''
128 '''
129 Add code
129 Add code
130 '''
130 '''
131 self.isConfig = True
131 self.isConfig = True
132 # print("Setup Done")
132 # print("Setup Done")
133 pass
133 pass
134
134
135
135
136 def readAMISRHeader(self,fp):
136 def readAMISRHeader(self,fp):
137
137
138 if self.isConfig and (not self.flagNoMoreFiles):
138 if self.isConfig and (not self.flagNoMoreFiles):
139 newShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
139 newShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
140 if self.dataShape != newShape and newShape != None:
140 if self.dataShape != newShape and newShape != None:
141 raise schainpy.admin.SchainError("NEW FILE HAS A DIFFERENT SHAPE: ")
141 raise schainpy.admin.SchainError("NEW FILE HAS A DIFFERENT SHAPE: ")
142 print(self.dataShape,newShape,"\n")
142 print(self.dataShape,newShape,"\n")
143 return 0
143 return 0
144 else:
144 else:
145 self.dataShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
145 self.dataShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
146
146
147
147
148 header = 'Raw11/Data/RadacHeader'
148 header = 'Raw11/Data/RadacHeader'
149 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
149 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
150 if (self.startDate> datetime.date(2021, 7, 15)) or self.flag_old_beams: #Se cambió la forma de extracción de Apuntes el 17 o forzar con flag de reorganización
150 if (self.startDate> datetime.date(2021, 7, 15)) or self.flag_old_beams: #Se cambió la forma de extracción de Apuntes el 17 o forzar con flag de reorganización
151 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
151 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
152 self.trueBeams = self.beamcodeFile.split("\n")
152 self.trueBeams = self.beamcodeFile.split("\n")
153 self.trueBeams.pop()#remove last
153 self.trueBeams.pop()#remove last
154 [self.realBeamCode.append(x) for x in self.trueBeams if x not in self.realBeamCode]
154 [self.realBeamCode.append(x) for x in self.trueBeams if x not in self.realBeamCode]
155 self.beamCode = [int(x, 16) for x in self.realBeamCode]
155 self.beamCode = [int(x, 16) for x in self.realBeamCode]
156 else:
156 else:
157 _beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
157 _beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
158 self.beamCode = _beamCode[0,:]
158 self.beamCode = _beamCode[0,:]
159
159
160 if self.beamCodeMap == None:
160 if self.beamCodeMap == None:
161 self.beamCodeMap = fp['Setup/BeamcodeMap']
161 self.beamCodeMap = fp['Setup/BeamcodeMap']
162 for beam in self.beamCode:
162 for beam in self.beamCode:
163 beamAziElev = numpy.where(self.beamCodeMap[:,0]==beam)
163 beamAziElev = numpy.where(self.beamCodeMap[:,0]==beam)
164 beamAziElev = beamAziElev[0].squeeze()
164 beamAziElev = beamAziElev[0].squeeze()
165 self.azimuthList.append(self.beamCodeMap[beamAziElev,1])
165 self.azimuthList.append(self.beamCodeMap[beamAziElev,1])
166 self.elevationList.append(self.beamCodeMap[beamAziElev,2])
166 self.elevationList.append(self.beamCodeMap[beamAziElev,2])
167 #print("Beamssss: ",self.beamCodeMap[beamAziElev,1],self.beamCodeMap[beamAziElev,2])
167 #print("Beamssss: ",self.beamCodeMap[beamAziElev,1],self.beamCodeMap[beamAziElev,2])
168 #print(self.beamCode)
168 #print(self.beamCode)
169 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
169 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
170 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
170 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
171 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
171 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
172 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
172 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
173 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
173 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
174 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
174 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
175 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
175 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
176 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
176 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
177 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
177 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
178 self.frequency = fp.get('Rx/Frequency')
178 self.frequency = fp.get('Rx/Frequency')
179 txAus = fp.get('Raw11/Data/Pulsewidth')
179 txAus = fp.get('Raw11/Data/Pulsewidth')
180
180
181
181
182 self.nblocks = self.pulseCount.shape[0] #nblocks
182 self.nblocks = self.pulseCount.shape[0] #nblocks
183
183
184 self.nprofiles = self.pulseCount.shape[1] #nprofile
184 self.nprofiles = self.pulseCount.shape[1] #nprofile
185 self.nsa = self.nsamplesPulse[0,0] #ngates
185 self.nsa = self.nsamplesPulse[0,0] #ngates
186 self.nchannels = len(self.beamCode)
186 self.nchannels = len(self.beamCode)
187 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
187 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
188 #print("IPPS secs: ",self.ippSeconds)
188 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
189 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
189 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
190 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
190
191
191 #filling radar controller header parameters
192 #filling radar controller header parameters
192 self.__ippKm = self.ippSeconds *.15*1e6 # in km
193 self.__ippKm = self.ippSeconds *.15*1e6 # in km
193 self.__txA = (txAus[()])*.15 #(ipp[us]*.15km/1us) in km
194 self.__txA = (txAus[()])*.15 #(ipp[us]*.15km/1us) in km
194 self.__txB = 0
195 self.__txB = 0
195 nWindows=1
196 nWindows=1
196 self.__nSamples = self.nsa
197 self.__nSamples = self.nsa
197 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
198 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
198 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
199 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
199 #print("amisr-ipp:",self.ippSeconds, self.__ippKm)
200 #print("amisr-ipp:",self.ippSeconds, self.__ippKm)
200 #for now until understand why the code saved is different (code included even though code not in tuf file)
201 #for now until understand why the code saved is different (code included even though code not in tuf file)
201 #self.__codeType = 0
202 #self.__codeType = 0
202 # self.__nCode = None
203 # self.__nCode = None
203 # self.__nBaud = None
204 # self.__nBaud = None
204 self.__code = self.code
205 self.__code = self.code
205 self.__codeType = 0
206 self.__codeType = 0
206 if self.code != None:
207 if self.code != None:
207 self.__codeType = 1
208 self.__codeType = 1
208 self.__nCode = self.nCode
209 self.__nCode = self.nCode
209 self.__nBaud = self.nBaud
210 self.__nBaud = self.nBaud
210 #self.__code = 0
211 #self.__code = 0
211
212
212 #filling system header parameters
213 #filling system header parameters
213 self.__nSamples = self.nsa
214 self.__nSamples = self.nsa
214 self.newProfiles = self.nprofiles/self.nchannels
215 self.newProfiles = self.nprofiles/self.nchannels
215 self.__channelList = list(range(self.nchannels))
216 self.__channelList = list(range(self.nchannels))
216
217
217 self.__frequency = self.frequency[0][0]
218 self.__frequency = self.frequency[0][0]
218
219
219
220
220 return 1
221 return 1
221
222
222
223
223 def createBuffers(self):
224 def createBuffers(self):
224
225
225 pass
226 pass
226
227
227 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
228 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
228 self.path = path
229 self.path = path
229 self.startDate = startDate
230 self.startDate = startDate
230 self.endDate = endDate
231 self.endDate = endDate
231 self.startTime = startTime
232 self.startTime = startTime
232 self.endTime = endTime
233 self.endTime = endTime
233 self.walk = walk
234 self.walk = walk
234
235
235 def __checkPath(self):
236 def __checkPath(self):
236 if os.path.exists(self.path):
237 if os.path.exists(self.path):
237 self.status = 1
238 self.status = 1
238 else:
239 else:
239 self.status = 0
240 self.status = 0
240 print('Path:%s does not exists'%self.path)
241 print('Path:%s does not exists'%self.path)
241
242
242 return
243 return
243
244
244
245
245 def __selDates(self, amisr_dirname_format):
246 def __selDates(self, amisr_dirname_format):
246 try:
247 try:
247 year = int(amisr_dirname_format[0:4])
248 year = int(amisr_dirname_format[0:4])
248 month = int(amisr_dirname_format[4:6])
249 month = int(amisr_dirname_format[4:6])
249 dom = int(amisr_dirname_format[6:8])
250 dom = int(amisr_dirname_format[6:8])
250 thisDate = datetime.date(year,month,dom)
251 thisDate = datetime.date(year,month,dom)
251 #margen de un día extra, igual luego se filtra for fecha y hora
252 #margen de un día extra, igual luego se filtra for fecha y hora
252 if (thisDate>=(self.startDate - datetime.timedelta(days=self.margin_days)) and thisDate <= (self.endDate)+ datetime.timedelta(days=1)):
253 if (thisDate>=(self.startDate - datetime.timedelta(days=self.margin_days)) and thisDate <= (self.endDate)+ datetime.timedelta(days=1)):
253 return amisr_dirname_format
254 return amisr_dirname_format
254 except:
255 except:
255 return None
256 return None
256
257
257
258
258 def __findDataForDates(self,online=False):
259 def __findDataForDates(self,online=False):
259
260
260 if not(self.status):
261 if not(self.status):
261 return None
262 return None
262
263
263 pat = '\d+.\d+'
264 pat = '\d+.\d+'
264 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
265 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
265 dirnameList = [x for x in dirnameList if x!=None]
266 dirnameList = [x for x in dirnameList if x!=None]
266 dirnameList = [x.string for x in dirnameList]
267 dirnameList = [x.string for x in dirnameList]
267 if not(online):
268 if not(online):
268 dirnameList = [self.__selDates(x) for x in dirnameList]
269 dirnameList = [self.__selDates(x) for x in dirnameList]
269 dirnameList = [x for x in dirnameList if x!=None]
270 dirnameList = [x for x in dirnameList if x!=None]
270 if len(dirnameList)>0:
271 if len(dirnameList)>0:
271 self.status = 1
272 self.status = 1
272 self.dirnameList = dirnameList
273 self.dirnameList = dirnameList
273 self.dirnameList.sort()
274 self.dirnameList.sort()
274 else:
275 else:
275 self.status = 0
276 self.status = 0
276 return None
277 return None
277
278
278 def __getTimeFromData(self):
279 def __getTimeFromData(self):
279 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
280 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
280 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
281 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
281
282
282 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
283 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
283 print('........................................')
284 print('........................................')
284 filter_filenameList = []
285 filter_filenameList = []
285 self.filenameList.sort()
286 self.filenameList.sort()
286 total_files = len(self.filenameList)
287 total_files = len(self.filenameList)
287 #for i in range(len(self.filenameList)-1):
288 #for i in range(len(self.filenameList)-1):
288 for i in range(total_files):
289 for i in range(total_files):
289 filename = self.filenameList[i]
290 filename = self.filenameList[i]
290 #print("file-> ",filename)
291 #print("file-> ",filename)
291 try:
292 try:
292 fp = h5py.File(filename,'r')
293 fp = h5py.File(filename,'r')
293 time_str = fp.get('Time/RadacTimeString')
294 time_str = fp.get('Time/RadacTimeString')
294
295
295 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
296 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
296 #startDateTimeStr_File = "2019-12-16 09:21:11"
297 #startDateTimeStr_File = "2019-12-16 09:21:11"
297 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
298 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
298 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
299 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
299
300
300 #endDateTimeStr_File = "2019-12-16 11:10:11"
301 #endDateTimeStr_File = "2019-12-16 11:10:11"
301 endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
302 endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
302 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
303 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
303 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
304 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
304
305
305 fp.close()
306 fp.close()
306
307
307 #print("check time", startDateTime_File)
308 #print("check time", startDateTime_File)
308 if self.timezone == 'lt':
309 if self.timezone == 'lt':
309 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
310 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
310 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
311 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
311 if (startDateTime_File >=startDateTime_Reader and endDateTime_File<=endDateTime_Reader):
312 if (startDateTime_File >=startDateTime_Reader and endDateTime_File<=endDateTime_Reader):
312 filter_filenameList.append(filename)
313 filter_filenameList.append(filename)
313
314
314 if (startDateTime_File>endDateTime_Reader):
315 if (startDateTime_File>endDateTime_Reader):
315 break
316 break
316 except Exception as e:
317 except Exception as e:
317 log.warning("Error opening file {} -> {}".format(os.path.split(filename)[1],e))
318 log.warning("Error opening file {} -> {}".format(os.path.split(filename)[1],e))
318
319
319 filter_filenameList.sort()
320 filter_filenameList.sort()
320 self.filenameList = filter_filenameList
321 self.filenameList = filter_filenameList
321
322
322 return 1
323 return 1
323
324
324 def __filterByGlob1(self, dirName):
325 def __filterByGlob1(self, dirName):
325 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
326 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
326 filter_files.sort()
327 filter_files.sort()
327 filterDict = {}
328 filterDict = {}
328 filterDict.setdefault(dirName)
329 filterDict.setdefault(dirName)
329 filterDict[dirName] = filter_files
330 filterDict[dirName] = filter_files
330 return filterDict
331 return filterDict
331
332
332 def __getFilenameList(self, fileListInKeys, dirList):
333 def __getFilenameList(self, fileListInKeys, dirList):
333 for value in fileListInKeys:
334 for value in fileListInKeys:
334 dirName = list(value.keys())[0]
335 dirName = list(value.keys())[0]
335 for file in value[dirName]:
336 for file in value[dirName]:
336 filename = os.path.join(dirName, file)
337 filename = os.path.join(dirName, file)
337 self.filenameList.append(filename)
338 self.filenameList.append(filename)
338
339
339
340
340 def __selectDataForTimes(self, online=False):
341 def __selectDataForTimes(self, online=False):
341 #aun no esta implementado el filtro for tiempo
342 #aun no esta implementado el filtro for tiempo
342 if not(self.status):
343 if not(self.status):
343 return None
344 return None
344
345
345 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
346 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
346 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
347 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
347 self.__getFilenameList(fileListInKeys, dirList)
348 self.__getFilenameList(fileListInKeys, dirList)
348 if not(online):
349 if not(online):
349 #filtro por tiempo
350 #filtro por tiempo
350 if not(self.all):
351 if not(self.all):
351 self.__getTimeFromData()
352 self.__getTimeFromData()
352
353
353 if len(self.filenameList)>0:
354 if len(self.filenameList)>0:
354 self.status = 1
355 self.status = 1
355 self.filenameList.sort()
356 self.filenameList.sort()
356 else:
357 else:
357 self.status = 0
358 self.status = 0
358 return None
359 return None
359
360
360 else:
361 else:
361 #get the last file - 1
362 #get the last file - 1
362 self.filenameList = [self.filenameList[-2]]
363 self.filenameList = [self.filenameList[-2]]
363 new_dirnameList = []
364 new_dirnameList = []
364 for dirname in self.dirnameList:
365 for dirname in self.dirnameList:
365 junk = numpy.array([dirname in x for x in self.filenameList])
366 junk = numpy.array([dirname in x for x in self.filenameList])
366 junk_sum = junk.sum()
367 junk_sum = junk.sum()
367 if junk_sum > 0:
368 if junk_sum > 0:
368 new_dirnameList.append(dirname)
369 new_dirnameList.append(dirname)
369 self.dirnameList = new_dirnameList
370 self.dirnameList = new_dirnameList
370 return 1
371 return 1
371
372
372 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
373 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
373 endTime=datetime.time(23,59,59),walk=True):
374 endTime=datetime.time(23,59,59),walk=True):
374
375
375 if endDate ==None:
376 if endDate ==None:
376 startDate = datetime.datetime.utcnow().date()
377 startDate = datetime.datetime.utcnow().date()
377 endDate = datetime.datetime.utcnow().date()
378 endDate = datetime.datetime.utcnow().date()
378
379
379 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
380 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
380
381
381 self.__checkPath()
382 self.__checkPath()
382
383
383 self.__findDataForDates(online=True)
384 self.__findDataForDates(online=True)
384
385
385 self.dirnameList = [self.dirnameList[-1]]
386 self.dirnameList = [self.dirnameList[-1]]
386
387
387 self.__selectDataForTimes(online=True)
388 self.__selectDataForTimes(online=True)
388
389
389 return
390 return
390
391
391
392
392 def searchFilesOffLine(self,
393 def searchFilesOffLine(self,
393 path,
394 path,
394 startDate,
395 startDate,
395 endDate,
396 endDate,
396 startTime=datetime.time(0,0,0),
397 startTime=datetime.time(0,0,0),
397 endTime=datetime.time(23,59,59),
398 endTime=datetime.time(23,59,59),
398 walk=True):
399 walk=True):
399
400
400 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
401 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
401
402
402 self.__checkPath()
403 self.__checkPath()
403
404
404 self.__findDataForDates()
405 self.__findDataForDates()
405
406
406 self.__selectDataForTimes()
407 self.__selectDataForTimes()
407
408
408 for i in range(len(self.filenameList)):
409 for i in range(len(self.filenameList)):
409 print("%s" %(self.filenameList[i]))
410 print("%s" %(self.filenameList[i]))
410
411
411 return
412 return
412
413
413 def __setNextFileOffline(self):
414 def __setNextFileOffline(self):
414
415
415 try:
416 try:
416 self.filename = self.filenameList[self.fileIndex]
417 self.filename = self.filenameList[self.fileIndex]
417 self.amisrFilePointer = h5py.File(self.filename,'r')
418 self.amisrFilePointer = h5py.File(self.filename,'r')
418 self.fileIndex += 1
419 self.fileIndex += 1
419 except:
420 except:
420 self.flagNoMoreFiles = 1
421 self.flagNoMoreFiles = 1
421 raise schainpy.admin.SchainError('No more files to read')
422 raise schainpy.admin.SchainError('No more files to read')
422 return 0
423 return 0
423
424
424 self.flagIsNewFile = 1
425 self.flagIsNewFile = 1
425 print("Setting the file: %s"%self.filename)
426 print("Setting the file: %s"%self.filename)
426
427
427 return 1
428 return 1
428
429
429
430
430 def __setNextFileOnline(self):
431 def __setNextFileOnline(self):
431 filename = self.filenameList[0]
432 filename = self.filenameList[0]
432 if self.__filename_online != None:
433 if self.__filename_online != None:
433 self.__selectDataForTimes(online=True)
434 self.__selectDataForTimes(online=True)
434 filename = self.filenameList[0]
435 filename = self.filenameList[0]
435 wait = 0
436 wait = 0
436 self.__waitForNewFile=300 ## DEBUG:
437 self.__waitForNewFile=300 ## DEBUG:
437 while self.__filename_online == filename:
438 while self.__filename_online == filename:
438 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
439 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
439 if wait == 5:
440 if wait == 5:
440 self.flagNoMoreFiles = 1
441 self.flagNoMoreFiles = 1
441 return 0
442 return 0
442 sleep(self.__waitForNewFile)
443 sleep(self.__waitForNewFile)
443 self.__selectDataForTimes(online=True)
444 self.__selectDataForTimes(online=True)
444 filename = self.filenameList[0]
445 filename = self.filenameList[0]
445 wait += 1
446 wait += 1
446
447
447 self.__filename_online = filename
448 self.__filename_online = filename
448
449
449 self.amisrFilePointer = h5py.File(filename,'r')
450 self.amisrFilePointer = h5py.File(filename,'r')
450 self.flagIsNewFile = 1
451 self.flagIsNewFile = 1
451 self.filename = filename
452 self.filename = filename
452 print("Setting the file: %s"%self.filename)
453 print("Setting the file: %s"%self.filename)
453 return 1
454 return 1
454
455
455
456
456 def readData(self):
457 def readData(self):
457 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
458 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
458 re = buffer[:,:,:,0]
459 re = buffer[:,:,:,0]
459 im = buffer[:,:,:,1]
460 im = buffer[:,:,:,1]
460 dataset = re + im*1j
461 dataset = re + im*1j
461
462
462 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
463 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
463 timeset = self.radacTime[:,0]
464 timeset = self.radacTime[:,0]
464
465
465 return dataset,timeset
466 return dataset,timeset
466
467
467 def reshapeData(self):
468 def reshapeData(self):
468 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
469 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
469 channels = self.beamCodeByPulse[0,:]
470 channels = self.beamCodeByPulse[0,:]
470 nchan = self.nchannels
471 nchan = self.nchannels
471 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
472 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
472 nblocks = self.nblocks
473 nblocks = self.nblocks
473 nsamples = self.nsa
474 nsamples = self.nsa
474
475
475 #Dimensions : nChannels, nProfiles, nSamples
476 #Dimensions : nChannels, nProfiles, nSamples
476 new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
477 new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
477 ############################################
478 ############################################
478
479
479 for thisChannel in range(nchan):
480 for thisChannel in range(nchan):
480 new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[thisChannel])[0],:]
481 new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[thisChannel])[0],:]
481
482
482
483
483 new_block = numpy.transpose(new_block, (1,0,2,3))
484 new_block = numpy.transpose(new_block, (1,0,2,3))
484 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
485 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
485
486
486 return new_block
487 return new_block
487
488
488 def updateIndexes(self):
489 def updateIndexes(self):
489
490
490 pass
491 pass
491
492
492 def fillJROHeader(self):
493 def fillJROHeader(self):
493
494
494 #fill radar controller header
495 #fill radar controller header
495 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
496 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
496 txA=self.__txA,
497 txA=self.__txA,
497 txB=0,
498 txB=0,
498 nWindows=1,
499 nWindows=1,
499 nHeights=self.__nSamples,
500 nHeights=self.__nSamples,
500 firstHeight=self.__firstHeight,
501 firstHeight=self.__firstHeight,
501 deltaHeight=self.__deltaHeight,
502 deltaHeight=self.__deltaHeight,
502 codeType=self.__codeType,
503 codeType=self.__codeType,
503 nCode=self.__nCode, nBaud=self.__nBaud,
504 nCode=self.__nCode, nBaud=self.__nBaud,
504 code = self.__code,
505 code = self.__code,
505 fClock=1)
506 fClock=1)
506 #fill system header
507 #fill system header
507 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
508 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
508 nProfiles=self.newProfiles,
509 nProfiles=self.newProfiles,
509 nChannels=len(self.__channelList),
510 nChannels=len(self.__channelList),
510 adcResolution=14,
511 adcResolution=14,
511 pciDioBusWidth=32)
512 pciDioBusWidth=32)
512
513
513 self.dataOut.type = "Voltage"
514 self.dataOut.type = "Voltage"
514 self.dataOut.data = None
515 self.dataOut.data = None
515 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
516 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
516 # self.dataOut.nChannels = 0
517 # self.dataOut.nChannels = 0
517
518
518 # self.dataOut.nHeights = 0
519 # self.dataOut.nHeights = 0
519
520
520 self.dataOut.nProfiles = self.newProfiles*self.nblocks
521 self.dataOut.nProfiles = self.newProfiles*self.nblocks
521 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
522 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
522 ranges = numpy.reshape(self.rangeFromFile[()],(-1))
523 ranges = numpy.reshape(self.rangeFromFile[()],(-1))
523 self.dataOut.heightList = ranges/1000.0 #km
524 self.dataOut.heightList = ranges/1000.0 #km
524 self.dataOut.channelList = self.__channelList
525 self.dataOut.channelList = self.__channelList
525 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
526 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
526
527
527 # self.dataOut.channelIndexList = None
528 # self.dataOut.channelIndexList = None
528
529
529
530
530 self.dataOut.azimuthList = numpy.array(self.azimuthList)
531 self.dataOut.azimuthList = numpy.array(self.azimuthList)
531 self.dataOut.elevationList = numpy.array(self.elevationList)
532 self.dataOut.elevationList = numpy.array(self.elevationList)
532 self.dataOut.codeList = numpy.array(self.beamCode)
533 self.dataOut.codeList = numpy.array(self.beamCode)
533 #print(self.dataOut.elevationList)
534 #print(self.dataOut.elevationList)
534 self.dataOut.flagNoData = True
535 self.dataOut.flagNoData = True
535
536
536 #Set to TRUE if the data is discontinuous
537 #Set to TRUE if the data is discontinuous
537 self.dataOut.flagDiscontinuousBlock = False
538 self.dataOut.flagDiscontinuousBlock = False
538
539
539 self.dataOut.utctime = None
540 self.dataOut.utctime = None
540
541
541 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
542 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
542 if self.timezone == 'lt':
543 if self.timezone == 'lt':
543 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
544 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
544 else:
545 else:
545 self.dataOut.timeZone = 0 #by default time is UTC
546 self.dataOut.timeZone = 0 #by default time is UTC
546
547
547 self.dataOut.dstFlag = 0
548 self.dataOut.dstFlag = 0
548 self.dataOut.errorCount = 0
549 self.dataOut.errorCount = 0
549 self.dataOut.nCohInt = 1
550 self.dataOut.nCohInt = 1
550 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
551 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
551 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
552 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
552 self.dataOut.flagShiftFFT = False
553 self.dataOut.flagShiftFFT = False
553 self.dataOut.ippSeconds = self.ippSeconds
554 self.dataOut.ippSeconds = self.ippSeconds
554
555
555 #Time interval between profiles
556 #Time interval between profiles
556 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
557 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
557
558
558 self.dataOut.frequency = self.__frequency
559 self.dataOut.frequency = self.__frequency
559 self.dataOut.realtime = self.online
560 self.dataOut.realtime = self.online
560 pass
561 pass
561
562
562 def readNextFile(self,online=False):
563 def readNextFile(self,online=False):
563
564
564 if not(online):
565 if not(online):
565 newFile = self.__setNextFileOffline()
566 newFile = self.__setNextFileOffline()
566 else:
567 else:
567 newFile = self.__setNextFileOnline()
568 newFile = self.__setNextFileOnline()
568
569
569 if not(newFile):
570 if not(newFile):
570 self.dataOut.error = True
571 self.dataOut.error = True
571 return 0
572 return 0
572
573
573 if not self.readAMISRHeader(self.amisrFilePointer):
574 if not self.readAMISRHeader(self.amisrFilePointer):
574 self.dataOut.error = True
575 self.dataOut.error = True
575 return 0
576 return 0
576
577
577 self.createBuffers()
578 self.createBuffers()
578 self.fillJROHeader()
579 self.fillJROHeader()
579
580
580 #self.__firstFile = False
581 #self.__firstFile = False
581
582
582
583
583
584
584 self.dataset,self.timeset = self.readData()
585 self.dataset,self.timeset = self.readData()
585
586
586 if self.endDate!=None:
587 if self.endDate!=None:
587 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
588 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
588 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
589 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
589 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
590 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
590 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
591 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
591 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
592 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
592 if self.timezone == 'lt':
593 if self.timezone == 'lt':
593 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
594 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
594 if (startDateTime_File>endDateTime_Reader):
595 if (startDateTime_File>endDateTime_Reader):
595 return 0
596 return 0
596
597
597 self.jrodataset = self.reshapeData()
598 self.jrodataset = self.reshapeData()
598 #----self.updateIndexes()
599 #----self.updateIndexes()
599 self.profileIndex = 0
600 self.profileIndex = 0
600
601
601 return 1
602 return 1
602
603
603
604
604 def __hasNotDataInBuffer(self):
605 def __hasNotDataInBuffer(self):
605 if self.profileIndex >= (self.newProfiles*self.nblocks):
606 if self.profileIndex >= (self.newProfiles*self.nblocks):
606 return 1
607 return 1
607 return 0
608 return 0
608
609
609
610
610 def getData(self):
611 def getData(self):
611
612
612 if self.flagNoMoreFiles:
613 if self.flagNoMoreFiles:
613 self.dataOut.flagNoData = True
614 self.dataOut.flagNoData = True
614 return 0
615 return 0
615
616
616 if self.__hasNotDataInBuffer():
617 if self.__hasNotDataInBuffer():
617 if not (self.readNextFile(self.online)):
618 if not (self.readNextFile(self.online)):
618 return 0
619 return 0
619
620
620
621
621 if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
622 if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
622 self.dataOut.flagNoData = True
623 self.dataOut.flagNoData = True
623 return 0
624 return 0
624
625
625 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
626 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
626
627
627 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
628 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
628
629
629 #print("R_t",self.timeset)
630 #print("R_t",self.timeset)
630
631
631 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
632 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
632 #verificar basic header de jro data y ver si es compatible con este valor
633 #verificar basic header de jro data y ver si es compatible con este valor
633 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
634 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
634 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
635 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
635 indexblock = self.profileIndex/self.newProfiles
636 indexblock = self.profileIndex/self.newProfiles
636 #print (indexblock, indexprof)
637 #print (indexblock, indexprof)
637 diffUTC = 0
638 diffUTC = 0
638 t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
639 t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
639
640
640 #print("utc :",indexblock," __ ",t_comp)
641 #print("utc :",indexblock," __ ",t_comp)
641 #print(numpy.shape(self.timeset))
642 #print(numpy.shape(self.timeset))
642 self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
643 self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
643 #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
644 #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
644
645
645 self.dataOut.profileIndex = self.profileIndex
646 self.dataOut.profileIndex = self.profileIndex
646 #print("N profile:",self.profileIndex,self.newProfiles,self.nblocks,self.dataOut.utctime)
647 #print("N profile:",self.profileIndex,self.newProfiles,self.nblocks,self.dataOut.utctime)
647 self.dataOut.flagNoData = False
648 self.dataOut.flagNoData = False
648 # if indexprof == 0:
649 # if indexprof == 0:
649 # print("kamisr: ",self.dataOut.utctime)
650 # print("kamisr: ",self.dataOut.utctime)
650
651
651 self.profileIndex += 1
652 self.profileIndex += 1
652
653
653 return self.dataOut.data #retorno necesario??
654 return self.dataOut.data #retorno necesario??
654
655
655
656
656 def run(self, **kwargs):
657 def run(self, **kwargs):
657 '''
658 '''
658 This method will be called many times so here you should put all your code
659 This method will be called many times so here you should put all your code
659 '''
660 '''
660 #print("running kamisr")
661 #print("running kamisr")
661 if not self.isConfig:
662 if not self.isConfig:
662 self.setup(**kwargs)
663 self.setup(**kwargs)
663 self.isConfig = True
664 self.isConfig = True
664
665
665 self.getData()
666 self.getData()
Show file before | Show file after | Hide comments
@@ -1,1890 +1,1948
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """Spectra processing Unit and operations
5 """Spectra processing Unit and operations
6
6
7 Here you will find the processing unit `SpectraProc` and several operations
7 Here you will find the processing unit `SpectraProc` and several operations
8 to work with Spectra data type
8 to work with Spectra data type
9 """
9 """
10
10
11 import time
11 import time
12 import itertools
12 import itertools
13
13
14 import numpy
14 import numpy
15 import math
15 import math
16
16
17 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
17 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
18 from schainpy.model.data.jrodata import Spectra
18 from schainpy.model.data.jrodata import Spectra
19 from schainpy.model.data.jrodata import hildebrand_sekhon
19 from schainpy.model.data.jrodata import hildebrand_sekhon
20 from schainpy.model.data import _noise
20 from schainpy.model.data import _noise
21
21
22 from schainpy.utils import log
22 from schainpy.utils import log
23
23 import matplotlib.pyplot as plt
24 from scipy.optimize import curve_fit
24 from scipy.optimize import curve_fit
25
25
26 class SpectraProc(ProcessingUnit):
26 class SpectraProc(ProcessingUnit):
27
27
28 def __init__(self):
28 def __init__(self):
29
29
30 ProcessingUnit.__init__(self)
30 ProcessingUnit.__init__(self)
31
31
32 self.buffer = None
32 self.buffer = None
33 self.firstdatatime = None
33 self.firstdatatime = None
34 self.profIndex = 0
34 self.profIndex = 0
35 self.dataOut = Spectra()
35 self.dataOut = Spectra()
36 self.id_min = None
36 self.id_min = None
37 self.id_max = None
37 self.id_max = None
38 self.setupReq = False #Agregar a todas las unidades de proc
38 self.setupReq = False #Agregar a todas las unidades de proc
39
39
40 def __updateSpecFromVoltage(self):
40 def __updateSpecFromVoltage(self):
41
41
42
42
43
43
44 self.dataOut.timeZone = self.dataIn.timeZone
44 self.dataOut.timeZone = self.dataIn.timeZone
45 self.dataOut.dstFlag = self.dataIn.dstFlag
45 self.dataOut.dstFlag = self.dataIn.dstFlag
46 self.dataOut.errorCount = self.dataIn.errorCount
46 self.dataOut.errorCount = self.dataIn.errorCount
47 self.dataOut.useLocalTime = self.dataIn.useLocalTime
47 self.dataOut.useLocalTime = self.dataIn.useLocalTime
48 try:
48 try:
49 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
49 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
50 except:
50 except:
51 pass
51 pass
52 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
52 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
53 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
53 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
54 self.dataOut.channelList = self.dataIn.channelList
54 self.dataOut.channelList = self.dataIn.channelList
55 self.dataOut.heightList = self.dataIn.heightList
55 self.dataOut.heightList = self.dataIn.heightList
56 self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
56 self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
57 self.dataOut.nProfiles = self.dataOut.nFFTPoints
57 self.dataOut.nProfiles = self.dataOut.nFFTPoints
58 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
58 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
59 self.dataOut.utctime = self.firstdatatime
59 self.dataOut.utctime = self.firstdatatime
60 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
60 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
61 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
61 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
62 self.dataOut.flagShiftFFT = False
62 self.dataOut.flagShiftFFT = False
63 self.dataOut.nCohInt = self.dataIn.nCohInt
63 self.dataOut.nCohInt = self.dataIn.nCohInt
64 self.dataOut.nIncohInt = 1
64 self.dataOut.nIncohInt = 1
65 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
65 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
66 self.dataOut.frequency = self.dataIn.frequency
66 self.dataOut.frequency = self.dataIn.frequency
67 self.dataOut.realtime = self.dataIn.realtime
67 self.dataOut.realtime = self.dataIn.realtime
68 self.dataOut.azimuth = self.dataIn.azimuth
68 self.dataOut.azimuth = self.dataIn.azimuth
69 self.dataOut.zenith = self.dataIn.zenith
69 self.dataOut.zenith = self.dataIn.zenith
70 self.dataOut.codeList = self.dataIn.codeList
70 self.dataOut.codeList = self.dataIn.codeList
71 self.dataOut.azimuthList = self.dataIn.azimuthList
71 self.dataOut.azimuthList = self.dataIn.azimuthList
72 self.dataOut.elevationList = self.dataIn.elevationList
72 self.dataOut.elevationList = self.dataIn.elevationList
73
73
74
74
75 def __getFft(self):
75 def __getFft(self):
76 """
76 """
77 Convierte valores de Voltaje a Spectra
77 Convierte valores de Voltaje a Spectra
78
78
79 Affected:
79 Affected:
80 self.dataOut.data_spc
80 self.dataOut.data_spc
81 self.dataOut.data_cspc
81 self.dataOut.data_cspc
82 self.dataOut.data_dc
82 self.dataOut.data_dc
83 self.dataOut.heightList
83 self.dataOut.heightList
84 self.profIndex
84 self.profIndex
85 self.buffer
85 self.buffer
86 self.dataOut.flagNoData
86 self.dataOut.flagNoData
87 """
87 """
88 fft_volt = numpy.fft.fft(
88 fft_volt = numpy.fft.fft(
89 self.buffer, n=self.dataOut.nFFTPoints, axis=1)
89 self.buffer, n=self.dataOut.nFFTPoints, axis=1)
90 fft_volt = fft_volt.astype(numpy.dtype('complex'))
90 fft_volt = fft_volt.astype(numpy.dtype('complex'))
91 dc = fft_volt[:, 0, :]
91 dc = fft_volt[:, 0, :]
92
92
93 # calculo de self-spectra
93 # calculo de self-spectra
94 fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
94 fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
95 spc = fft_volt * numpy.conjugate(fft_volt)
95 spc = fft_volt * numpy.conjugate(fft_volt)
96 spc = spc.real
96 spc = spc.real
97
97
98 blocksize = 0
98 blocksize = 0
99 blocksize += dc.size
99 blocksize += dc.size
100 blocksize += spc.size
100 blocksize += spc.size
101
101
102 cspc = None
102 cspc = None
103 pairIndex = 0
103 pairIndex = 0
104 if self.dataOut.pairsList != None:
104 if self.dataOut.pairsList != None:
105 # calculo de cross-spectra
105 # calculo de cross-spectra
106 cspc = numpy.zeros(
106 cspc = numpy.zeros(
107 (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
107 (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
108 for pair in self.dataOut.pairsList:
108 for pair in self.dataOut.pairsList:
109 if pair[0] not in self.dataOut.channelList:
109 if pair[0] not in self.dataOut.channelList:
110 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
110 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
111 str(pair), str(self.dataOut.channelList)))
111 str(pair), str(self.dataOut.channelList)))
112 if pair[1] not in self.dataOut.channelList:
112 if pair[1] not in self.dataOut.channelList:
113 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
113 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
114 str(pair), str(self.dataOut.channelList)))
114 str(pair), str(self.dataOut.channelList)))
115
115
116 cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
116 cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
117 numpy.conjugate(fft_volt[pair[1], :, :])
117 numpy.conjugate(fft_volt[pair[1], :, :])
118 pairIndex += 1
118 pairIndex += 1
119 blocksize += cspc.size
119 blocksize += cspc.size
120
120
121 self.dataOut.data_spc = spc
121 self.dataOut.data_spc = spc
122 self.dataOut.data_cspc = cspc
122 self.dataOut.data_cspc = cspc
123 self.dataOut.data_dc = dc
123 self.dataOut.data_dc = dc
124 self.dataOut.blockSize = blocksize
124 self.dataOut.blockSize = blocksize
125 self.dataOut.flagShiftFFT = False
125 self.dataOut.flagShiftFFT = False
126
126
127 def run(self, nProfiles=None, nFFTPoints=None, pairsList=None, ippFactor=None, shift_fft=False):
127 def run(self, nProfiles=None, nFFTPoints=None, pairsList=None, ippFactor=None, shift_fft=False):
128
128 #print("run spc proc")
129 if self.dataIn.type == "Spectra":
129 if self.dataIn.type == "Spectra":
130
130
131 try:
131 try:
132 self.dataOut.copy(self.dataIn)
132 self.dataOut.copy(self.dataIn)
133
133
134 except Exception as e:
134 except Exception as e:
135 print("Error dataIn ",e)
135 print("Error dataIn ",e)
136
136
137 if shift_fft:
137 if shift_fft:
138 #desplaza a la derecha en el eje 2 determinadas posiciones
138 #desplaza a la derecha en el eje 2 determinadas posiciones
139 shift = int(self.dataOut.nFFTPoints/2)
139 shift = int(self.dataOut.nFFTPoints/2)
140 self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
140 self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
141
141
142 if self.dataOut.data_cspc is not None:
142 if self.dataOut.data_cspc is not None:
143 #desplaza a la derecha en el eje 2 determinadas posiciones
143 #desplaza a la derecha en el eje 2 determinadas posiciones
144 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
144 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
145 if pairsList:
145 if pairsList:
146 self.__selectPairs(pairsList)
146 self.__selectPairs(pairsList)
147
147
148
148
149 elif self.dataIn.type == "Voltage":
149 elif self.dataIn.type == "Voltage":
150
150
151 self.dataOut.flagNoData = True
151 self.dataOut.flagNoData = True
152
152
153 if nFFTPoints == None:
153 if nFFTPoints == None:
154 raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
154 raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
155
155
156 if nProfiles == None:
156 if nProfiles == None:
157 nProfiles = nFFTPoints
157 nProfiles = nFFTPoints
158
158
159 if ippFactor == None:
159 if ippFactor == None:
160 self.dataOut.ippFactor = 1
160 self.dataOut.ippFactor = 1
161
161
162 self.dataOut.nFFTPoints = nFFTPoints
162 self.dataOut.nFFTPoints = nFFTPoints
163 #print(" volts ch,prof, h: ", self.dataIn.data.shape)
163 #print(" volts ch,prof, h: ", self.dataIn.data.shape)
164 if self.buffer is None:
164 if self.buffer is None:
165 self.buffer = numpy.zeros((self.dataIn.nChannels,
165 self.buffer = numpy.zeros((self.dataIn.nChannels,
166 nProfiles,
166 nProfiles,
167 self.dataIn.nHeights),
167 self.dataIn.nHeights),
168 dtype='complex')
168 dtype='complex')
169
169
170 if self.dataIn.flagDataAsBlock:
170 if self.dataIn.flagDataAsBlock:
171 nVoltProfiles = self.dataIn.data.shape[1]
171 nVoltProfiles = self.dataIn.data.shape[1]
172
172
173 if nVoltProfiles == nProfiles:
173 if nVoltProfiles == nProfiles:
174 self.buffer = self.dataIn.data.copy()
174 self.buffer = self.dataIn.data.copy()
175 self.profIndex = nVoltProfiles
175 self.profIndex = nVoltProfiles
176
176
177 elif nVoltProfiles < nProfiles:
177 elif nVoltProfiles < nProfiles:
178
178
179 if self.profIndex == 0:
179 if self.profIndex == 0:
180 self.id_min = 0
180 self.id_min = 0
181 self.id_max = nVoltProfiles
181 self.id_max = nVoltProfiles
182
182
183 self.buffer[:, self.id_min:self.id_max,
183 self.buffer[:, self.id_min:self.id_max,
184 :] = self.dataIn.data
184 :] = self.dataIn.data
185 self.profIndex += nVoltProfiles
185 self.profIndex += nVoltProfiles
186 self.id_min += nVoltProfiles
186 self.id_min += nVoltProfiles
187 self.id_max += nVoltProfiles
187 self.id_max += nVoltProfiles
188 else:
188 else:
189 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
189 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
190 self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
190 self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
191 self.dataOut.flagNoData = True
191 self.dataOut.flagNoData = True
192 else:
192 else:
193 self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
193 self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
194 self.profIndex += 1
194 self.profIndex += 1
195
195
196 if self.firstdatatime == None:
196 if self.firstdatatime == None:
197 self.firstdatatime = self.dataIn.utctime
197 self.firstdatatime = self.dataIn.utctime
198
198
199 if self.profIndex == nProfiles:
199 if self.profIndex == nProfiles:
200
200
201 self.__updateSpecFromVoltage()
201 self.__updateSpecFromVoltage()
202
202 if pairsList == None:
203 if pairsList == None:
203 self.dataOut.pairsList = [pair for pair in itertools.combinations(self.dataOut.channelList, 2)]
204 self.dataOut.pairsList = [pair for pair in itertools.combinations(self.dataOut.channelList, 2)]
204 else:
205 else:
205 self.dataOut.pairsList = pairsList
206 self.dataOut.pairsList = pairsList
206 self.__getFft()
207 self.__getFft()
207 self.dataOut.flagNoData = False
208 self.dataOut.flagNoData = False
208 self.firstdatatime = None
209 self.firstdatatime = None
209 self.profIndex = 0
210 self.profIndex = 0
210
211
212
211 else:
213 else:
212 raise ValueError("The type of input object '%s' is not valid".format(
214 raise ValueError("The type of input object '%s' is not valid".format(
213 self.dataIn.type))
215 self.dataIn.type))
214
216
215
217
216 def __selectPairs(self, pairsList):
218 def __selectPairs(self, pairsList):
217
219
218 if not pairsList:
220 if not pairsList:
219 return
221 return
220
222
221 pairs = []
223 pairs = []
222 pairsIndex = []
224 pairsIndex = []
223
225
224 for pair in pairsList:
226 for pair in pairsList:
225 if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
227 if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
226 continue
228 continue
227 pairs.append(pair)
229 pairs.append(pair)
228 pairsIndex.append(pairs.index(pair))
230 pairsIndex.append(pairs.index(pair))
229
231
230 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
232 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
231 self.dataOut.pairsList = pairs
233 self.dataOut.pairsList = pairs
232
234
233 return
235 return
234
236
235 def selectFFTs(self, minFFT, maxFFT ):
237 def selectFFTs(self, minFFT, maxFFT ):
236 """
238 """
237 Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
239 Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
238 minFFT<= FFT <= maxFFT
240 minFFT<= FFT <= maxFFT
239 """
241 """
240
242
241 if (minFFT > maxFFT):
243 if (minFFT > maxFFT):
242 raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT))
244 raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT))
243
245
244 if (minFFT < self.dataOut.getFreqRange()[0]):
246 if (minFFT < self.dataOut.getFreqRange()[0]):
245 minFFT = self.dataOut.getFreqRange()[0]
247 minFFT = self.dataOut.getFreqRange()[0]
246
248
247 if (maxFFT > self.dataOut.getFreqRange()[-1]):
249 if (maxFFT > self.dataOut.getFreqRange()[-1]):
248 maxFFT = self.dataOut.getFreqRange()[-1]
250 maxFFT = self.dataOut.getFreqRange()[-1]
249
251
250 minIndex = 0
252 minIndex = 0
251 maxIndex = 0
253 maxIndex = 0
252 FFTs = self.dataOut.getFreqRange()
254 FFTs = self.dataOut.getFreqRange()
253
255
254 inda = numpy.where(FFTs >= minFFT)
256 inda = numpy.where(FFTs >= minFFT)
255 indb = numpy.where(FFTs <= maxFFT)
257 indb = numpy.where(FFTs <= maxFFT)
256
258
257 try:
259 try:
258 minIndex = inda[0][0]
260 minIndex = inda[0][0]
259 except:
261 except:
260 minIndex = 0
262 minIndex = 0
261
263
262 try:
264 try:
263 maxIndex = indb[0][-1]
265 maxIndex = indb[0][-1]
264 except:
266 except:
265 maxIndex = len(FFTs)
267 maxIndex = len(FFTs)
266
268
267 self.selectFFTsByIndex(minIndex, maxIndex)
269 self.selectFFTsByIndex(minIndex, maxIndex)
268
270
269 return 1
271 return 1
270
272
271 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
273 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
272 newheis = numpy.where(
274 newheis = numpy.where(
273 self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
275 self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
274
276
275 if hei_ref != None:
277 if hei_ref != None:
276 newheis = numpy.where(self.dataOut.heightList > hei_ref)
278 newheis = numpy.where(self.dataOut.heightList > hei_ref)
277
279
278 minIndex = min(newheis[0])
280 minIndex = min(newheis[0])
279 maxIndex = max(newheis[0])
281 maxIndex = max(newheis[0])
280 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
282 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
281 heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
283 heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
282
284
283 # determina indices
285 # determina indices
284 nheis = int(self.dataOut.radarControllerHeaderObj.txB /
286 nheis = int(self.dataOut.radarControllerHeaderObj.txB /
285 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
287 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
286 avg_dB = 10 * \
288 avg_dB = 10 * \
287 numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
289 numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
288 beacon_dB = numpy.sort(avg_dB)[-nheis:]
290 beacon_dB = numpy.sort(avg_dB)[-nheis:]
289 beacon_heiIndexList = []
291 beacon_heiIndexList = []
290 for val in avg_dB.tolist():
292 for val in avg_dB.tolist():
291 if val >= beacon_dB[0]:
293 if val >= beacon_dB[0]:
292 beacon_heiIndexList.append(avg_dB.tolist().index(val))
294 beacon_heiIndexList.append(avg_dB.tolist().index(val))
293
295
294 #data_spc = data_spc[:,:,beacon_heiIndexList]
296 #data_spc = data_spc[:,:,beacon_heiIndexList]
295 data_cspc = None
297 data_cspc = None
296 if self.dataOut.data_cspc is not None:
298 if self.dataOut.data_cspc is not None:
297 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
299 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
298 #data_cspc = data_cspc[:,:,beacon_heiIndexList]
300 #data_cspc = data_cspc[:,:,beacon_heiIndexList]
299
301
300 data_dc = None
302 data_dc = None
301 if self.dataOut.data_dc is not None:
303 if self.dataOut.data_dc is not None:
302 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
304 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
303 #data_dc = data_dc[:,beacon_heiIndexList]
305 #data_dc = data_dc[:,beacon_heiIndexList]
304
306
305 self.dataOut.data_spc = data_spc
307 self.dataOut.data_spc = data_spc
306 self.dataOut.data_cspc = data_cspc
308 self.dataOut.data_cspc = data_cspc
307 self.dataOut.data_dc = data_dc
309 self.dataOut.data_dc = data_dc
308 self.dataOut.heightList = heightList
310 self.dataOut.heightList = heightList
309 self.dataOut.beacon_heiIndexList = beacon_heiIndexList
311 self.dataOut.beacon_heiIndexList = beacon_heiIndexList
310
312
311 return 1
313 return 1
312
314
313 def selectFFTsByIndex(self, minIndex, maxIndex):
315 def selectFFTsByIndex(self, minIndex, maxIndex):
314 """
316 """
315
317
316 """
318 """
317
319
318 if (minIndex < 0) or (minIndex > maxIndex):
320 if (minIndex < 0) or (minIndex > maxIndex):
319 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (minIndex, maxIndex))
321 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (minIndex, maxIndex))
320
322
321 if (maxIndex >= self.dataOut.nProfiles):
323 if (maxIndex >= self.dataOut.nProfiles):
322 maxIndex = self.dataOut.nProfiles-1
324 maxIndex = self.dataOut.nProfiles-1
323
325
324 #Spectra
326 #Spectra
325 data_spc = self.dataOut.data_spc[:,minIndex:maxIndex+1,:]
327 data_spc = self.dataOut.data_spc[:,minIndex:maxIndex+1,:]
326
328
327 data_cspc = None
329 data_cspc = None
328 if self.dataOut.data_cspc is not None:
330 if self.dataOut.data_cspc is not None:
329 data_cspc = self.dataOut.data_cspc[:,minIndex:maxIndex+1,:]
331 data_cspc = self.dataOut.data_cspc[:,minIndex:maxIndex+1,:]
330
332
331 data_dc = None
333 data_dc = None
332 if self.dataOut.data_dc is not None:
334 if self.dataOut.data_dc is not None:
333 data_dc = self.dataOut.data_dc[minIndex:maxIndex+1,:]
335 data_dc = self.dataOut.data_dc[minIndex:maxIndex+1,:]
334
336
335 self.dataOut.data_spc = data_spc
337 self.dataOut.data_spc = data_spc
336 self.dataOut.data_cspc = data_cspc
338 self.dataOut.data_cspc = data_cspc
337 self.dataOut.data_dc = data_dc
339 self.dataOut.data_dc = data_dc
338
340
339 self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1])
341 self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1])
340 self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1]
342 self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1]
341 self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1]
343 self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1]
342
344
343 return 1
345 return 1
344
346
345 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
347 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
346 # validacion de rango
348 # validacion de rango
347 if minHei == None:
349 if minHei == None:
348 minHei = self.dataOut.heightList[0]
350 minHei = self.dataOut.heightList[0]
349
351
350 if maxHei == None:
352 if maxHei == None:
351 maxHei = self.dataOut.heightList[-1]
353 maxHei = self.dataOut.heightList[-1]
352
354
353 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
355 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
354 print('minHei: %.2f is out of the heights range' % (minHei))
356 print('minHei: %.2f is out of the heights range' % (minHei))
355 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
357 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
356 minHei = self.dataOut.heightList[0]
358 minHei = self.dataOut.heightList[0]
357
359
358 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
360 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
359 print('maxHei: %.2f is out of the heights range' % (maxHei))
361 print('maxHei: %.2f is out of the heights range' % (maxHei))
360 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
362 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
361 maxHei = self.dataOut.heightList[-1]
363 maxHei = self.dataOut.heightList[-1]
362
364
363 # validacion de velocidades
365 # validacion de velocidades
364 velrange = self.dataOut.getVelRange(1)
366 velrange = self.dataOut.getVelRange(1)
365
367
366 if minVel == None:
368 if minVel == None:
367 minVel = velrange[0]
369 minVel = velrange[0]
368
370
369 if maxVel == None:
371 if maxVel == None:
370 maxVel = velrange[-1]
372 maxVel = velrange[-1]
371
373
372 if (minVel < velrange[0]) or (minVel > maxVel):
374 if (minVel < velrange[0]) or (minVel > maxVel):
373 print('minVel: %.2f is out of the velocity range' % (minVel))
375 print('minVel: %.2f is out of the velocity range' % (minVel))
374 print('minVel is setting to %.2f' % (velrange[0]))
376 print('minVel is setting to %.2f' % (velrange[0]))
375 minVel = velrange[0]
377 minVel = velrange[0]
376
378
377 if (maxVel > velrange[-1]) or (maxVel < minVel):
379 if (maxVel > velrange[-1]) or (maxVel < minVel):
378 print('maxVel: %.2f is out of the velocity range' % (maxVel))
380 print('maxVel: %.2f is out of the velocity range' % (maxVel))
379 print('maxVel is setting to %.2f' % (velrange[-1]))
381 print('maxVel is setting to %.2f' % (velrange[-1]))
380 maxVel = velrange[-1]
382 maxVel = velrange[-1]
381
383
382 # seleccion de indices para rango
384 # seleccion de indices para rango
383 minIndex = 0
385 minIndex = 0
384 maxIndex = 0
386 maxIndex = 0
385 heights = self.dataOut.heightList
387 heights = self.dataOut.heightList
386
388
387 inda = numpy.where(heights >= minHei)
389 inda = numpy.where(heights >= minHei)
388 indb = numpy.where(heights <= maxHei)
390 indb = numpy.where(heights <= maxHei)
389
391
390 try:
392 try:
391 minIndex = inda[0][0]
393 minIndex = inda[0][0]
392 except:
394 except:
393 minIndex = 0
395 minIndex = 0
394
396
395 try:
397 try:
396 maxIndex = indb[0][-1]
398 maxIndex = indb[0][-1]
397 except:
399 except:
398 maxIndex = len(heights)
400 maxIndex = len(heights)
399
401
400 if (minIndex < 0) or (minIndex > maxIndex):
402 if (minIndex < 0) or (minIndex > maxIndex):
401 raise ValueError("some value in (%d,%d) is not valid" % (
403 raise ValueError("some value in (%d,%d) is not valid" % (
402 minIndex, maxIndex))
404 minIndex, maxIndex))
403
405
404 if (maxIndex >= self.dataOut.nHeights):
406 if (maxIndex >= self.dataOut.nHeights):
405 maxIndex = self.dataOut.nHeights - 1
407 maxIndex = self.dataOut.nHeights - 1
406
408
407 # seleccion de indices para velocidades
409 # seleccion de indices para velocidades
408 indminvel = numpy.where(velrange >= minVel)
410 indminvel = numpy.where(velrange >= minVel)
409 indmaxvel = numpy.where(velrange <= maxVel)
411 indmaxvel = numpy.where(velrange <= maxVel)
410 try:
412 try:
411 minIndexVel = indminvel[0][0]
413 minIndexVel = indminvel[0][0]
412 except:
414 except:
413 minIndexVel = 0
415 minIndexVel = 0
414
416
415 try:
417 try:
416 maxIndexVel = indmaxvel[0][-1]
418 maxIndexVel = indmaxvel[0][-1]
417 except:
419 except:
418 maxIndexVel = len(velrange)
420 maxIndexVel = len(velrange)
419
421
420 # seleccion del espectro
422 # seleccion del espectro
421 data_spc = self.dataOut.data_spc[:,
423 data_spc = self.dataOut.data_spc[:,
422 minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
424 minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
423 # estimacion de ruido
425 # estimacion de ruido
424 noise = numpy.zeros(self.dataOut.nChannels)
426 noise = numpy.zeros(self.dataOut.nChannels)
425
427
426 for channel in range(self.dataOut.nChannels):
428 for channel in range(self.dataOut.nChannels):
427 daux = data_spc[channel, :, :]
429 daux = data_spc[channel, :, :]
428 sortdata = numpy.sort(daux, axis=None)
430 sortdata = numpy.sort(daux, axis=None)
429 noise[channel] = hildebrand_sekhon(sortdata, self.dataOut.nIncohInt)
431 noise[channel] = hildebrand_sekhon(sortdata, self.dataOut.nIncohInt)
430
432
431 self.dataOut.noise_estimation = noise.copy()
433 self.dataOut.noise_estimation = noise.copy()
432
434
433 return 1
435 return 1
434
436
435 class removeDC(Operation):
437 class removeDC(Operation):
436
438
437 def run(self, dataOut, mode=2):
439 def run(self, dataOut, mode=2):
438 self.dataOut = dataOut
440 self.dataOut = dataOut
439 jspectra = self.dataOut.data_spc
441 jspectra = self.dataOut.data_spc
440 jcspectra = self.dataOut.data_cspc
442 jcspectra = self.dataOut.data_cspc
441
443
442 num_chan = jspectra.shape[0]
444 num_chan = jspectra.shape[0]
443 num_hei = jspectra.shape[2]
445 num_hei = jspectra.shape[2]
444
446
445 if jcspectra is not None:
447 if jcspectra is not None:
446 jcspectraExist = True
448 jcspectraExist = True
447 num_pairs = jcspectra.shape[0]
449 num_pairs = jcspectra.shape[0]
448 else:
450 else:
449 jcspectraExist = False
451 jcspectraExist = False
450
452
451 freq_dc = int(jspectra.shape[1] / 2)
453 freq_dc = int(jspectra.shape[1] / 2)
452 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
454 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
453 ind_vel = ind_vel.astype(int)
455 ind_vel = ind_vel.astype(int)
454
456
455 if ind_vel[0] < 0:
457 if ind_vel[0] < 0:
456 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
458 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
457
459
458 if mode == 1:
460 if mode == 1:
459 jspectra[:, freq_dc, :] = (
461 jspectra[:, freq_dc, :] = (
460 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
462 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
461
463
462 if jcspectraExist:
464 if jcspectraExist:
463 jcspectra[:, freq_dc, :] = (
465 jcspectra[:, freq_dc, :] = (
464 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
466 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
465
467
466 if mode == 2:
468 if mode == 2:
467
469
468 vel = numpy.array([-2, -1, 1, 2])
470 vel = numpy.array([-2, -1, 1, 2])
469 xx = numpy.zeros([4, 4])
471 xx = numpy.zeros([4, 4])
470
472
471 for fil in range(4):
473 for fil in range(4):
472 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
474 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
473
475
474 xx_inv = numpy.linalg.inv(xx)
476 xx_inv = numpy.linalg.inv(xx)
475 xx_aux = xx_inv[0, :]
477 xx_aux = xx_inv[0, :]
476
478
477 for ich in range(num_chan):
479 for ich in range(num_chan):
478 yy = jspectra[ich, ind_vel, :]
480 yy = jspectra[ich, ind_vel, :]
479 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
481 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
480
482
481 junkid = jspectra[ich, freq_dc, :] <= 0
483 junkid = jspectra[ich, freq_dc, :] <= 0
482 cjunkid = sum(junkid)
484 cjunkid = sum(junkid)
483
485
484 if cjunkid.any():
486 if cjunkid.any():
485 jspectra[ich, freq_dc, junkid.nonzero()] = (
487 jspectra[ich, freq_dc, junkid.nonzero()] = (
486 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
488 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
487
489
488 if jcspectraExist:
490 if jcspectraExist:
489 for ip in range(num_pairs):
491 for ip in range(num_pairs):
490 yy = jcspectra[ip, ind_vel, :]
492 yy = jcspectra[ip, ind_vel, :]
491 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
493 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
492
494
493 self.dataOut.data_spc = jspectra
495 self.dataOut.data_spc = jspectra
494 self.dataOut.data_cspc = jcspectra
496 self.dataOut.data_cspc = jcspectra
495
497
496 return self.dataOut
498 return self.dataOut
497
499
498 class getNoise(Operation):
500 class getNoise(Operation):
499 warnings = False
501 warnings = False
500 def __init__(self):
502 def __init__(self):
501
503
502 Operation.__init__(self)
504 Operation.__init__(self)
503
505
504 def run(self, dataOut, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
506 def run(self, dataOut, minHei=None, maxHei=None,minVel=None, maxVel=None, minFreq= None, maxFreq=None, warnings=False):
505 self.dataOut = dataOut
507 self.dataOut = dataOut
506 self.warnings = warnings
508 self.warnings = warnings
507 if minHei == None:
509 if minHei == None:
508 minHei = self.dataOut.heightList[0]
510 minHei = self.dataOut.heightList[0]
509
511
510 if maxHei == None:
512 if maxHei == None:
511 maxHei = self.dataOut.heightList[-1]
513 maxHei = self.dataOut.heightList[-1]
512
514
513 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
515 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
514 if self.warnings:
516 if self.warnings:
515 print('minHei: %.2f is out of the heights range' % (minHei))
517 print('minHei: %.2f is out of the heights range' % (minHei))
516 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
518 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
517 minHei = self.dataOut.heightList[0]
519 minHei = self.dataOut.heightList[0]
518
520
519 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
521 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
520 if self.warnings:
522 if self.warnings:
521 print('maxHei: %.2f is out of the heights range' % (maxHei))
523 print('maxHei: %.2f is out of the heights range' % (maxHei))
522 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
524 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
523 maxHei = self.dataOut.heightList[-1]
525 maxHei = self.dataOut.heightList[-1]
524
526
525
527
526 #indices relativos a los puntos de fft, puede ser de acuerdo a velocidad o frecuencia
528 #indices relativos a los puntos de fft, puede ser de acuerdo a velocidad o frecuencia
527 minIndexFFT = 0
529 minIndexFFT = 0
528 maxIndexFFT = 0
530 maxIndexFFT = 0
529 # validacion de velocidades
531 # validacion de velocidades
530 indminPoint = None
532 indminPoint = None
531 indmaxPoint = None
533 indmaxPoint = None
532
534 if self.dataOut.type == 'Spectra':
533 if minVel == None and maxVel == None:
535 if minVel == None and maxVel == None :
534
536
535 freqrange = self.dataOut.getFreqRange(1)
537 freqrange = self.dataOut.getFreqRange(1)
536
538
537 if minFreq == None:
539 if minFreq == None:
538 minFreq = freqrange[0]
540 minFreq = freqrange[0]
539
541
540 if maxFreq == None:
542 if maxFreq == None:
541 maxFreq = freqrange[-1]
543 maxFreq = freqrange[-1]
542
544
543 if (minFreq < freqrange[0]) or (minFreq > maxFreq):
545 if (minFreq < freqrange[0]) or (minFreq > maxFreq):
544 if self.warnings:
546 if self.warnings:
545 print('minFreq: %.2f is out of the frequency range' % (minFreq))
547 print('minFreq: %.2f is out of the frequency range' % (minFreq))
546 print('minFreq is setting to %.2f' % (freqrange[0]))
548 print('minFreq is setting to %.2f' % (freqrange[0]))
547 minFreq = freqrange[0]
549 minFreq = freqrange[0]
548
550
549 if (maxFreq > freqrange[-1]) or (maxFreq < minFreq):
551 if (maxFreq > freqrange[-1]) or (maxFreq < minFreq):
550 if self.warnings:
552 if self.warnings:
551 print('maxFreq: %.2f is out of the frequency range' % (maxFreq))
553 print('maxFreq: %.2f is out of the frequency range' % (maxFreq))
552 print('maxFreq is setting to %.2f' % (freqrange[-1]))
554 print('maxFreq is setting to %.2f' % (freqrange[-1]))
553 maxFreq = freqrange[-1]
555 maxFreq = freqrange[-1]
554
556
555 indminPoint = numpy.where(freqrange >= minFreq)
557 indminPoint = numpy.where(freqrange >= minFreq)
556 indmaxPoint = numpy.where(freqrange <= maxFreq)
558 indmaxPoint = numpy.where(freqrange <= maxFreq)
557
559
558 else:
560 else:
561
559 velrange = self.dataOut.getVelRange(1)
562 velrange = self.dataOut.getVelRange(1)
560
563
561 if minVel == None:
564 if minVel == None:
562 minVel = velrange[0]
565 minVel = velrange[0]
563
566
564 if maxVel == None:
567 if maxVel == None:
565 maxVel = velrange[-1]
568 maxVel = velrange[-1]
566
569
567 if (minVel < velrange[0]) or (minVel > maxVel):
570 if (minVel < velrange[0]) or (minVel > maxVel):
568 if self.warnings:
571 if self.warnings:
569 print('minVel: %.2f is out of the velocity range' % (minVel))
572 print('minVel: %.2f is out of the velocity range' % (minVel))
570 print('minVel is setting to %.2f' % (velrange[0]))
573 print('minVel is setting to %.2f' % (velrange[0]))
571 minVel = velrange[0]
574 minVel = velrange[0]
572
575
573 if (maxVel > velrange[-1]) or (maxVel < minVel):
576 if (maxVel > velrange[-1]) or (maxVel < minVel):
574 if self.warnings:
577 if self.warnings:
575 print('maxVel: %.2f is out of the velocity range' % (maxVel))
578 print('maxVel: %.2f is out of the velocity range' % (maxVel))
576 print('maxVel is setting to %.2f' % (velrange[-1]))
579 print('maxVel is setting to %.2f' % (velrange[-1]))
577 maxVel = velrange[-1]
580 maxVel = velrange[-1]
578
581
579 indminPoint = numpy.where(velrange >= minVel)
582 indminPoint = numpy.where(velrange >= minVel)
580 indmaxPoint = numpy.where(velrange <= maxVel)
583 indmaxPoint = numpy.where(velrange <= maxVel)
581
584
582
585
583 # seleccion de indices para rango
586 # seleccion de indices para rango
584 minIndex = 0
587 minIndex = 0
585 maxIndex = 0
588 maxIndex = 0
586 heights = self.dataOut.heightList
589 heights = self.dataOut.heightList
587
590
588 inda = numpy.where(heights >= minHei)
591 inda = numpy.where(heights >= minHei)
589 indb = numpy.where(heights <= maxHei)
592 indb = numpy.where(heights <= maxHei)
590
593
591 try:
594 try:
592 minIndex = inda[0][0]
595 minIndex = inda[0][0]
593 except:
596 except:
594 minIndex = 0
597 minIndex = 0
595
598
596 try:
599 try:
597 maxIndex = indb[0][-1]
600 maxIndex = indb[0][-1]
598 except:
601 except:
599 maxIndex = len(heights)
602 maxIndex = len(heights)
600
603
601 if (minIndex < 0) or (minIndex > maxIndex):
604 if (minIndex < 0) or (minIndex > maxIndex):
602 raise ValueError("some value in (%d,%d) is not valid" % (
605 raise ValueError("some value in (%d,%d) is not valid" % (
603 minIndex, maxIndex))
606 minIndex, maxIndex))
604
607
605 if (maxIndex >= self.dataOut.nHeights):
608 if (maxIndex >= self.dataOut.nHeights):
606 maxIndex = self.dataOut.nHeights - 1
609 maxIndex = self.dataOut.nHeights - 1
607 #############################################################3
610 #############################################################3
608 # seleccion de indices para velocidades
611 # seleccion de indices para velocidades
609
612 if self.dataOut.type == 'Spectra':
610 try:
613 try:
611 minIndexFFT = indminPoint[0][0]
614 minIndexFFT = indminPoint[0][0]
612 except:
615 except:
613 minIndexFFT = 0
616 minIndexFFT = 0
614
617
615 try:
618 try:
616 maxIndexFFT = indmaxPoint[0][-1]
619 maxIndexFFT = indmaxPoint[0][-1]
617 except:
620 except:
618 maxIndexFFT = len( self.dataOut.getFreqRange(1))
621 maxIndexFFT = len( self.dataOut.getFreqRange(1))
619
622
620 #print(minIndex, maxIndex,minIndexVel, maxIndexVel)
623
621 self.dataOut.noise_estimation = None
624 self.dataOut.noise_estimation = None
625 noise = None
626 if self.dataOut.type == 'Voltage':
627 noise = self.dataOut.getNoise(ymin_index=minIndex, ymax_index=maxIndex)
628 #print(minIndex, maxIndex,minIndexVel, maxIndexVel)
629 elif self.dataOut.type == 'Spectra':
630 noise = self.dataOut.getNoise(xmin_index=minIndexFFT, xmax_index=maxIndexFFT, ymin_index=minIndex, ymax_index=maxIndex)
631 else:
622 noise = self.dataOut.getNoise(xmin_index=minIndexFFT, xmax_index=maxIndexFFT, ymin_index=minIndex, ymax_index=maxIndex)
632 noise = self.dataOut.getNoise(xmin_index=minIndexFFT, xmax_index=maxIndexFFT, ymin_index=minIndex, ymax_index=maxIndex)
623
624 self.dataOut.noise_estimation = noise.copy() # dataOut.noise
633 self.dataOut.noise_estimation = noise.copy() # dataOut.noise
625 #print("2: ",10*numpy.log10(self.dataOut.noise_estimation/64))
634 #print("2: ",10*numpy.log10(self.dataOut.noise_estimation/64))
635
626 return self.dataOut
636 return self.dataOut
627
637
628
638
629
639
630 # import matplotlib.pyplot as plt
640 # import matplotlib.pyplot as plt
631
641
632 def fit_func( x, a0, a1, a2): #, a3, a4, a5):
642 def fit_func( x, a0, a1, a2): #, a3, a4, a5):
633 z = (x - a1) / a2
643 z = (x - a1) / a2
634 y = a0 * numpy.exp(-z**2 / a2) #+ a3 + a4 * x + a5 * x**2
644 y = a0 * numpy.exp(-z**2 / a2) #+ a3 + a4 * x + a5 * x**2
635 return y
645 return y
636
646
637
647
638 class CleanRayleigh(Operation):
648 class CleanRayleigh(Operation):
639
649
640 def __init__(self):
650 def __init__(self):
641
651
642 Operation.__init__(self)
652 Operation.__init__(self)
643 self.i=0
653 self.i=0
644 self.isConfig = False
654 self.isConfig = False
645 self.__dataReady = False
655 self.__dataReady = False
646 self.__profIndex = 0
656 self.__profIndex = 0
647 self.byTime = False
657 self.byTime = False
648 self.byProfiles = False
658 self.byProfiles = False
649
659
650 self.bloques = None
660 self.bloques = None
651 self.bloque0 = None
661 self.bloque0 = None
652
662
653 self.index = 0
663 self.index = 0
654
664
655 self.buffer = 0
665 self.buffer = 0
656 self.buffer2 = 0
666 self.buffer2 = 0
657 self.buffer3 = 0
667 self.buffer3 = 0
658
668
659
669
660 def setup(self,dataOut,min_hei,max_hei,n, timeInterval,factor_stdv):
670 def setup(self,dataOut,min_hei,max_hei,n, timeInterval,factor_stdv):
661
671
662 self.nChannels = dataOut.nChannels
672 self.nChannels = dataOut.nChannels
663 self.nProf = dataOut.nProfiles
673 self.nProf = dataOut.nProfiles
664 self.nPairs = dataOut.data_cspc.shape[0]
674 self.nPairs = dataOut.data_cspc.shape[0]
665 self.pairsArray = numpy.array(dataOut.pairsList)
675 self.pairsArray = numpy.array(dataOut.pairsList)
666 self.spectra = dataOut.data_spc
676 self.spectra = dataOut.data_spc
667 self.cspectra = dataOut.data_cspc
677 self.cspectra = dataOut.data_cspc
668 self.heights = dataOut.heightList #alturas totales
678 self.heights = dataOut.heightList #alturas totales
669 self.nHeights = len(self.heights)
679 self.nHeights = len(self.heights)
670 self.min_hei = min_hei
680 self.min_hei = min_hei
671 self.max_hei = max_hei
681 self.max_hei = max_hei
672 if (self.min_hei == None):
682 if (self.min_hei == None):
673 self.min_hei = 0
683 self.min_hei = 0
674 if (self.max_hei == None):
684 if (self.max_hei == None):
675 self.max_hei = dataOut.heightList[-1]
685 self.max_hei = dataOut.heightList[-1]
676 self.hval = ((self.max_hei>=self.heights) & (self.heights >= self.min_hei)).nonzero()
686 self.hval = ((self.max_hei>=self.heights) & (self.heights >= self.min_hei)).nonzero()
677 self.heightsClean = self.heights[self.hval] #alturas filtradas
687 self.heightsClean = self.heights[self.hval] #alturas filtradas
678 self.hval = self.hval[0] # forma (N,), an solo N elementos -> Indices de alturas
688 self.hval = self.hval[0] # forma (N,), an solo N elementos -> Indices de alturas
679 self.nHeightsClean = len(self.heightsClean)
689 self.nHeightsClean = len(self.heightsClean)
680 self.channels = dataOut.channelList
690 self.channels = dataOut.channelList
681 self.nChan = len(self.channels)
691 self.nChan = len(self.channels)
682 self.nIncohInt = dataOut.nIncohInt
692 self.nIncohInt = dataOut.nIncohInt
683 self.__initime = dataOut.utctime
693 self.__initime = dataOut.utctime
684 self.maxAltInd = self.hval[-1]+1
694 self.maxAltInd = self.hval[-1]+1
685 self.minAltInd = self.hval[0]
695 self.minAltInd = self.hval[0]
686
696
687 self.crosspairs = dataOut.pairsList
697 self.crosspairs = dataOut.pairsList
688 self.nPairs = len(self.crosspairs)
698 self.nPairs = len(self.crosspairs)
689 self.normFactor = dataOut.normFactor
699 self.normFactor = dataOut.normFactor
690 self.nFFTPoints = dataOut.nFFTPoints
700 self.nFFTPoints = dataOut.nFFTPoints
691 self.ippSeconds = dataOut.ippSeconds
701 self.ippSeconds = dataOut.ippSeconds
692 self.currentTime = self.__initime
702 self.currentTime = self.__initime
693 self.pairsArray = numpy.array(dataOut.pairsList)
703 self.pairsArray = numpy.array(dataOut.pairsList)
694 self.factor_stdv = factor_stdv
704 self.factor_stdv = factor_stdv
695
705
696 if n != None :
706 if n != None :
697 self.byProfiles = True
707 self.byProfiles = True
698 self.nIntProfiles = n
708 self.nIntProfiles = n
699 else:
709 else:
700 self.__integrationtime = timeInterval
710 self.__integrationtime = timeInterval
701
711
702 self.__dataReady = False
712 self.__dataReady = False
703 self.isConfig = True
713 self.isConfig = True
704
714
705
715
706
716
707 def run(self, dataOut,min_hei=None,max_hei=None, n=None, timeInterval=10,factor_stdv=2.5):
717 def run(self, dataOut,min_hei=None,max_hei=None, n=None, timeInterval=10,factor_stdv=2.5):
708
718 #print("runing cleanRayleigh")
709 if not self.isConfig :
719 if not self.isConfig :
710
720
711 self.setup(dataOut, min_hei,max_hei,n,timeInterval,factor_stdv)
721 self.setup(dataOut, min_hei,max_hei,n,timeInterval,factor_stdv)
712
722
713 tini=dataOut.utctime
723 tini=dataOut.utctime
714
724
715 if self.byProfiles:
725 if self.byProfiles:
716 if self.__profIndex == self.nIntProfiles:
726 if self.__profIndex == self.nIntProfiles:
717 self.__dataReady = True
727 self.__dataReady = True
718 else:
728 else:
719 if (tini - self.__initime) >= self.__integrationtime:
729 if (tini - self.__initime) >= self.__integrationtime:
720
730
721 self.__dataReady = True
731 self.__dataReady = True
722 self.__initime = tini
732 self.__initime = tini
723
733
724 #if (tini.tm_min % 2) == 0 and (tini.tm_sec < 5 and self.fint==0):
734 #if (tini.tm_min % 2) == 0 and (tini.tm_sec < 5 and self.fint==0):
725
735
726 if self.__dataReady:
736 if self.__dataReady:
727
737
728 self.__profIndex = 0
738 self.__profIndex = 0
729 jspc = self.buffer
739 jspc = self.buffer
730 jcspc = self.buffer2
740 jcspc = self.buffer2
731 #jnoise = self.buffer3
741 #jnoise = self.buffer3
732 self.buffer = dataOut.data_spc
742 self.buffer = dataOut.data_spc
733 self.buffer2 = dataOut.data_cspc
743 self.buffer2 = dataOut.data_cspc
734 #self.buffer3 = dataOut.noise
744 #self.buffer3 = dataOut.noise
735 self.currentTime = dataOut.utctime
745 self.currentTime = dataOut.utctime
736 if numpy.any(jspc) :
746 if numpy.any(jspc) :
737 #print( jspc.shape, jcspc.shape)
747 #print( jspc.shape, jcspc.shape)
738 jspc = numpy.reshape(jspc,(int(len(jspc)/self.nChannels),self.nChannels,self.nFFTPoints,self.nHeights))
748 jspc = numpy.reshape(jspc,(int(len(jspc)/self.nChannels),self.nChannels,self.nFFTPoints,self.nHeights))
749 try:
739 jcspc= numpy.reshape(jcspc,(int(len(jcspc)/self.nPairs),self.nPairs,self.nFFTPoints,self.nHeights))
750 jcspc= numpy.reshape(jcspc,(int(len(jcspc)/self.nPairs),self.nPairs,self.nFFTPoints,self.nHeights))
751 except:
752 print("no cspc")
740 self.__dataReady = False
753 self.__dataReady = False
741 #print( jspc.shape, jcspc.shape)
754 #print( jspc.shape, jcspc.shape)
742 dataOut.flagNoData = False
755 dataOut.flagNoData = False
743 else:
756 else:
744 dataOut.flagNoData = True
757 dataOut.flagNoData = True
745 self.__dataReady = False
758 self.__dataReady = False
746 return dataOut
759 return dataOut
747 else:
760 else:
748 #print( len(self.buffer))
761 #print( len(self.buffer))
749 if numpy.any(self.buffer):
762 if numpy.any(self.buffer):
750 self.buffer = numpy.concatenate((self.buffer,dataOut.data_spc), axis=0)
763 self.buffer = numpy.concatenate((self.buffer,dataOut.data_spc), axis=0)
764 try:
751 self.buffer2 = numpy.concatenate((self.buffer2,dataOut.data_cspc), axis=0)
765 self.buffer2 = numpy.concatenate((self.buffer2,dataOut.data_cspc), axis=0)
752 self.buffer3 += dataOut.data_dc
766 self.buffer3 += dataOut.data_dc
767 except:
768 pass
753 else:
769 else:
754 self.buffer = dataOut.data_spc
770 self.buffer = dataOut.data_spc
755 self.buffer2 = dataOut.data_cspc
771 self.buffer2 = dataOut.data_cspc
756 self.buffer3 = dataOut.data_dc
772 self.buffer3 = dataOut.data_dc
757 #print self.index, self.fint
773 #print self.index, self.fint
758 #print self.buffer2.shape
774 #print self.buffer2.shape
759 dataOut.flagNoData = True ## NOTE: ?? revisar LUEGO
775 dataOut.flagNoData = True ## NOTE: ?? revisar LUEGO
760 self.__profIndex += 1
776 self.__profIndex += 1
761 return dataOut ## NOTE: REV
777 return dataOut ## NOTE: REV
762
778
763
779
764 #index = tini.tm_hour*12+tini.tm_min/5
780 #index = tini.tm_hour*12+tini.tm_min/5
765 '''REVISAR'''
781 '''
782 REVISAR
783 '''
766 # jspc = jspc/self.nFFTPoints/self.normFactor
784 # jspc = jspc/self.nFFTPoints/self.normFactor
767 # jcspc = jcspc/self.nFFTPoints/self.normFactor
785 # jcspc = jcspc/self.nFFTPoints/self.normFactor
768
786
769
787
770
788
771 tmp_spectra,tmp_cspectra = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
789 tmp_spectra,tmp_cspectra = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
772 dataOut.data_spc = tmp_spectra
790 dataOut.data_spc = tmp_spectra
773 dataOut.data_cspc = tmp_cspectra
791 dataOut.data_cspc = tmp_cspectra
774
792
775 #dataOut.data_spc,dataOut.data_cspc = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
793 #dataOut.data_spc,dataOut.data_cspc = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
776
794
777 dataOut.data_dc = self.buffer3
795 dataOut.data_dc = self.buffer3
778 dataOut.nIncohInt *= self.nIntProfiles
796 dataOut.nIncohInt *= self.nIntProfiles
797 dataOut.max_nIncohInt = self.nIntProfiles
779 dataOut.utctime = self.currentTime #tiempo promediado
798 dataOut.utctime = self.currentTime #tiempo promediado
780 #print("Time: ",time.localtime(dataOut.utctime))
799 #print("Time: ",time.localtime(dataOut.utctime))
781 # dataOut.data_spc = sat_spectra
800 # dataOut.data_spc = sat_spectra
782 # dataOut.data_cspc = sat_cspectra
801 # dataOut.data_cspc = sat_cspectra
783 self.buffer = 0
802 self.buffer = 0
784 self.buffer2 = 0
803 self.buffer2 = 0
785 self.buffer3 = 0
804 self.buffer3 = 0
786
805
787 return dataOut
806 return dataOut
788
807
789 def cleanRayleigh(self,dataOut,spectra,cspectra,factor_stdv):
808 def cleanRayleigh(self,dataOut,spectra,cspectra,factor_stdv):
790 #print("OP cleanRayleigh")
809 print("OP cleanRayleigh")
791 #import matplotlib.pyplot as plt
810 #import matplotlib.pyplot as plt
792 #for k in range(149):
811 #for k in range(149):
793 #channelsProcssd = []
812 #channelsProcssd = []
794 #channelA_ok = False
813 #channelA_ok = False
795 #rfunc = cspectra.copy() #self.bloques
814 #rfunc = cspectra.copy() #self.bloques
796 rfunc = spectra.copy()
815 rfunc = spectra.copy()
797 #rfunc = cspectra
816 #rfunc = cspectra
798 #val_spc = spectra*0.0 #self.bloque0*0.0
817 #val_spc = spectra*0.0 #self.bloque0*0.0
799 #val_cspc = cspectra*0.0 #self.bloques*0.0
818 #val_cspc = cspectra*0.0 #self.bloques*0.0
800 #in_sat_spectra = spectra.copy() #self.bloque0
819 #in_sat_spectra = spectra.copy() #self.bloque0
801 #in_sat_cspectra = cspectra.copy() #self.bloques
820 #in_sat_cspectra = cspectra.copy() #self.bloques
802
821
803
822
804 ###ONLY FOR TEST:
823 ###ONLY FOR TEST:
805 raxs = math.ceil(math.sqrt(self.nPairs))
824 raxs = math.ceil(math.sqrt(self.nPairs))
825 if raxs == 0:
826 raxs = 1
806 caxs = math.ceil(self.nPairs/raxs)
827 caxs = math.ceil(self.nPairs/raxs)
807 if self.nPairs <4:
828 if self.nPairs <4:
808 raxs = 2
829 raxs = 2
809 caxs = 2
830 caxs = 2
810 #print(raxs, caxs)
831 #print(raxs, caxs)
811 fft_rev = 14 #nFFT to plot
832 fft_rev = 14 #nFFT to plot
812 hei_rev = ((self.heights >= 550) & (self.heights <= 551)).nonzero() #hei to plot
833 hei_rev = ((self.heights >= 550) & (self.heights <= 551)).nonzero() #hei to plot
813 hei_rev = hei_rev[0]
834 hei_rev = hei_rev[0]
814 #print(hei_rev)
835 #print(hei_rev)
815
836
816 #print numpy.absolute(rfunc[:,0,0,14])
837 #print numpy.absolute(rfunc[:,0,0,14])
817
838
818 gauss_fit, covariance = None, None
839 gauss_fit, covariance = None, None
819 for ih in range(self.minAltInd,self.maxAltInd):
840 for ih in range(self.minAltInd,self.maxAltInd):
820 for ifreq in range(self.nFFTPoints):
841 for ifreq in range(self.nFFTPoints):
821 '''
842 '''
822 ###ONLY FOR TEST:
843 ###ONLY FOR TEST:
823 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
844 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
824 fig, axs = plt.subplots(raxs, caxs)
845 fig, axs = plt.subplots(raxs, caxs)
825 fig2, axs2 = plt.subplots(raxs, caxs)
846 fig2, axs2 = plt.subplots(raxs, caxs)
826 col_ax = 0
847 col_ax = 0
827 row_ax = 0
848 row_ax = 0
828 '''
849 '''
829 #print(self.nPairs)
850 #print(self.nPairs)
830 for ii in range(self.nChan): #PARES DE CANALES SELF y CROSS
851 for ii in range(self.nChan): #PARES DE CANALES SELF y CROSS
831 # if self.crosspairs[ii][1]-self.crosspairs[ii][0] > 1: # APLICAR SOLO EN PARES CONTIGUOS
852 # if self.crosspairs[ii][1]-self.crosspairs[ii][0] > 1: # APLICAR SOLO EN PARES CONTIGUOS
832 # continue
853 # continue
833 # if not self.crosspairs[ii][0] in channelsProcssd:
854 # if not self.crosspairs[ii][0] in channelsProcssd:
834 # channelA_ok = True
855 # channelA_ok = True
835 #print("pair: ",self.crosspairs[ii])
856 #print("pair: ",self.crosspairs[ii])
836 '''
857 '''
837 ###ONLY FOR TEST:
858 ###ONLY FOR TEST:
838 if (col_ax%caxs==0 and col_ax!=0 and self.nPairs !=1):
859 if (col_ax%caxs==0 and col_ax!=0 and self.nPairs !=1):
839 col_ax = 0
860 col_ax = 0
840 row_ax += 1
861 row_ax += 1
841 '''
862 '''
842 func2clean = 10*numpy.log10(numpy.absolute(rfunc[:,ii,ifreq,ih])) #Potencia?
863 func2clean = 10*numpy.log10(numpy.absolute(rfunc[:,ii,ifreq,ih])) #Potencia?
843 #print(func2clean.shape)
864 #print(func2clean.shape)
844 val = (numpy.isfinite(func2clean)==True).nonzero()
865 val = (numpy.isfinite(func2clean)==True).nonzero()
845
866
846 if len(val)>0: #limitador
867 if len(val)>0: #limitador
847 min_val = numpy.around(numpy.amin(func2clean)-2) #> (-40)
868 min_val = numpy.around(numpy.amin(func2clean)-2) #> (-40)
848 if min_val <= -40 :
869 if min_val <= -40 :
849 min_val = -40
870 min_val = -40
850 max_val = numpy.around(numpy.amax(func2clean)+2) #< 200
871 max_val = numpy.around(numpy.amax(func2clean)+2) #< 200
851 if max_val >= 200 :
872 if max_val >= 200 :
852 max_val = 200
873 max_val = 200
853 #print min_val, max_val
874 #print min_val, max_val
854 step = 1
875 step = 1
855 #print("Getting bins and the histogram")
876 #print("Getting bins and the histogram")
856 x_dist = min_val + numpy.arange(1 + ((max_val-(min_val))/step))*step
877 x_dist = min_val + numpy.arange(1 + ((max_val-(min_val))/step))*step
857 y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
878 y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
858 #print(len(y_dist),len(binstep[:-1]))
879 #print(len(y_dist),len(binstep[:-1]))
859 #print(row_ax,col_ax, " ..")
880 #print(row_ax,col_ax, " ..")
860 #print(self.pairsArray[ii][0],self.pairsArray[ii][1])
881 #print(self.pairsArray[ii][0],self.pairsArray[ii][1])
861 mean = numpy.sum(x_dist * y_dist) / numpy.sum(y_dist)
882 mean = numpy.sum(x_dist * y_dist) / numpy.sum(y_dist)
862 sigma = numpy.sqrt(numpy.sum(y_dist * (x_dist - mean)**2) / numpy.sum(y_dist))
883 sigma = numpy.sqrt(numpy.sum(y_dist * (x_dist - mean)**2) / numpy.sum(y_dist))
863 parg = [numpy.amax(y_dist),mean,sigma]
884 parg = [numpy.amax(y_dist),mean,sigma]
864
885
865 newY = None
886 newY = None
866
887
867 try :
888 try :
868 gauss_fit, covariance = curve_fit(fit_func, x_dist, y_dist,p0=parg)
889 gauss_fit, covariance = curve_fit(fit_func, x_dist, y_dist,p0=parg)
869 mode = gauss_fit[1]
890 mode = gauss_fit[1]
870 stdv = gauss_fit[2]
891 stdv = gauss_fit[2]
871 #print(" FIT OK",gauss_fit)
892 #print(" FIT OK",gauss_fit)
872 '''
893 '''
873 ###ONLY FOR TEST:
894 ###ONLY FOR TEST:
874 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
895 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
875 newY = fit_func(x_dist,gauss_fit[0],gauss_fit[1],gauss_fit[2])
896 newY = fit_func(x_dist,gauss_fit[0],gauss_fit[1],gauss_fit[2])
876 axs[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
897 axs[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
877 axs[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
898 axs[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
878 axs[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
899 axs[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
879 '''
900 '''
880 except:
901 except:
881 mode = mean
902 mode = mean
882 stdv = sigma
903 stdv = sigma
883 #print("FIT FAIL")
904 #print("FIT FAIL")
884 #continue
905 #continue
885
906
886
907
887 #print(mode,stdv)
908 #print(mode,stdv)
888 #Removing echoes greater than mode + std_factor*stdv
909 #Removing echoes greater than mode + std_factor*stdv
889 noval = (abs(func2clean - mode)>=(factor_stdv*stdv)).nonzero()
910 noval = (abs(func2clean - mode)>=(factor_stdv*stdv)).nonzero()
890 #noval tiene los indices que se van a remover
911 #noval tiene los indices que se van a remover
891 #print("Chan ",ii," novals: ",len(noval[0]))
912 #print("Chan ",ii," novals: ",len(noval[0]))
892 if len(noval[0]) > 0: #forma de array (N,) es igual a longitud (N)
913 if len(noval[0]) > 0: #forma de array (N,) es igual a longitud (N)
893 novall = ((func2clean - mode) >= (factor_stdv*stdv)).nonzero()
914 novall = ((func2clean - mode) >= (factor_stdv*stdv)).nonzero()
894 #print(novall)
915 #print(novall)
895 #print(" ",self.pairsArray[ii])
916 #print(" ",self.pairsArray[ii])
896 #cross_pairs = self.pairsArray[ii]
917 #cross_pairs = self.pairsArray[ii]
897 #Getting coherent echoes which are removed.
918 #Getting coherent echoes which are removed.
898 # if len(novall[0]) > 0:
919 # if len(novall[0]) > 0:
899 #
920 #
900 # val_spc[novall[0],cross_pairs[0],ifreq,ih] = 1
921 # val_spc[novall[0],cross_pairs[0],ifreq,ih] = 1
901 # val_spc[novall[0],cross_pairs[1],ifreq,ih] = 1
922 # val_spc[novall[0],cross_pairs[1],ifreq,ih] = 1
902 # val_cspc[novall[0],ii,ifreq,ih] = 1
923 # val_cspc[novall[0],ii,ifreq,ih] = 1
903 #print("OUT NOVALL 1")
924 #print("OUT NOVALL 1")
904 try:
925 try:
905 pair = (self.channels[ii],self.channels[ii + 1])
926 pair = (self.channels[ii],self.channels[ii + 1])
906 except:
927 except:
907 pair = (99,99)
928 pair = (99,99)
908 #print("par ", pair)
929 #print("par ", pair)
909 if ( pair in self.crosspairs):
930 if ( pair in self.crosspairs):
910 q = self.crosspairs.index(pair)
931 q = self.crosspairs.index(pair)
911 #print("está aqui: ", q, (ii,ii + 1))
932 #print("está aqui: ", q, (ii,ii + 1))
912 new_a = numpy.delete(cspectra[:,q,ifreq,ih], noval[0])
933 new_a = numpy.delete(cspectra[:,q,ifreq,ih], noval[0])
913 cspectra[noval,q,ifreq,ih] = numpy.mean(new_a) #mean CrossSpectra
934 cspectra[noval,q,ifreq,ih] = numpy.mean(new_a) #mean CrossSpectra
914
935
915 #if channelA_ok:
936 #if channelA_ok:
916 #chA = self.channels.index(cross_pairs[0])
937 #chA = self.channels.index(cross_pairs[0])
917 new_b = numpy.delete(spectra[:,ii,ifreq,ih], noval[0])
938 new_b = numpy.delete(spectra[:,ii,ifreq,ih], noval[0])
918 spectra[noval,ii,ifreq,ih] = numpy.mean(new_b) #mean Spectra Pair A
939 spectra[noval,ii,ifreq,ih] = numpy.mean(new_b) #mean Spectra Pair A
919 #channelA_ok = False
940 #channelA_ok = False
920
941
921 # chB = self.channels.index(cross_pairs[1])
942 # chB = self.channels.index(cross_pairs[1])
922 # new_c = numpy.delete(spectra[:,chB,ifreq,ih], noval[0])
943 # new_c = numpy.delete(spectra[:,chB,ifreq,ih], noval[0])
923 # spectra[noval,chB,ifreq,ih] = numpy.mean(new_c) #mean Spectra Pair B
944 # spectra[noval,chB,ifreq,ih] = numpy.mean(new_c) #mean Spectra Pair B
924 #
945 #
925 # channelsProcssd.append(self.crosspairs[ii][0]) # save channel A
946 # channelsProcssd.append(self.crosspairs[ii][0]) # save channel A
926 # channelsProcssd.append(self.crosspairs[ii][1]) # save channel B
947 # channelsProcssd.append(self.crosspairs[ii][1]) # save channel B
927 '''
948 '''
928 ###ONLY FOR TEST:
949 ###ONLY FOR TEST:
929 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
950 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
930 func2clean = 10*numpy.log10(numpy.absolute(spectra[:,ii,ifreq,ih]))
951 func2clean = 10*numpy.log10(numpy.absolute(spectra[:,ii,ifreq,ih]))
931 y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
952 y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
932 axs2[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
953 axs2[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
933 axs2[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
954 axs2[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
934 axs2[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
955 axs2[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
935 '''
956 '''
936 '''
957 '''
937 ###ONLY FOR TEST:
958 ###ONLY FOR TEST:
938 col_ax += 1 #contador de ploteo columnas
959 col_ax += 1 #contador de ploteo columnas
939 ##print(col_ax)
960 ##print(col_ax)
940 ###ONLY FOR TEST:
961 ###ONLY FOR TEST:
941 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
962 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
942 title = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km"
963 title = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km"
943 title2 = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km CLEANED"
964 title2 = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km CLEANED"
944 fig.suptitle(title)
965 fig.suptitle(title)
945 fig2.suptitle(title2)
966 fig2.suptitle(title2)
946 plt.show()
967 plt.show()
947 '''
968 '''
948 ##################################################################################################
969 ##################################################################################################
949
970
950 #print("Getting average of the spectra and cross-spectra from incoherent echoes.")
971 #print("Getting average of the spectra and cross-spectra from incoherent echoes.")
951 out_spectra = numpy.zeros([self.nChan,self.nFFTPoints,self.nHeights], dtype=float) #+numpy.nan
972 out_spectra = numpy.zeros([self.nChan,self.nFFTPoints,self.nHeights], dtype=float) #+numpy.nan
952 out_cspectra = numpy.zeros([self.nPairs,self.nFFTPoints,self.nHeights], dtype=complex) #+numpy.nan
973 out_cspectra = numpy.zeros([self.nPairs,self.nFFTPoints,self.nHeights], dtype=complex) #+numpy.nan
953 for ih in range(self.nHeights):
974 for ih in range(self.nHeights):
954 for ifreq in range(self.nFFTPoints):
975 for ifreq in range(self.nFFTPoints):
955 for ich in range(self.nChan):
976 for ich in range(self.nChan):
956 tmp = spectra[:,ich,ifreq,ih]
977 tmp = spectra[:,ich,ifreq,ih]
957 valid = (numpy.isfinite(tmp[:])==True).nonzero()
978 valid = (numpy.isfinite(tmp[:])==True).nonzero()
958
979
959 if len(valid[0]) >0 :
980 if len(valid[0]) >0 :
960 out_spectra[ich,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
981 out_spectra[ich,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
961
982
962 for icr in range(self.nPairs):
983 for icr in range(self.nPairs):
963 tmp = numpy.squeeze(cspectra[:,icr,ifreq,ih])
984 tmp = numpy.squeeze(cspectra[:,icr,ifreq,ih])
964 valid = (numpy.isfinite(tmp)==True).nonzero()
985 valid = (numpy.isfinite(tmp)==True).nonzero()
965 if len(valid[0]) > 0:
986 if len(valid[0]) > 0:
966 out_cspectra[icr,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
987 out_cspectra[icr,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
967
988
968 return out_spectra, out_cspectra
989 return out_spectra, out_cspectra
969
990
970 def REM_ISOLATED_POINTS(self,array,rth):
991 def REM_ISOLATED_POINTS(self,array,rth):
971 # import matplotlib.pyplot as plt
992 # import matplotlib.pyplot as plt
972 if rth == None :
993 if rth == None :
973 rth = 4
994 rth = 4
974 #print("REM ISO")
995 #print("REM ISO")
975 num_prof = len(array[0,:,0])
996 num_prof = len(array[0,:,0])
976 num_hei = len(array[0,0,:])
997 num_hei = len(array[0,0,:])
977 n2d = len(array[:,0,0])
998 n2d = len(array[:,0,0])
978
999
979 for ii in range(n2d) :
1000 for ii in range(n2d) :
980 #print ii,n2d
1001 #print ii,n2d
981 tmp = array[ii,:,:]
1002 tmp = array[ii,:,:]
982 #print tmp.shape, array[ii,101,:],array[ii,102,:]
1003 #print tmp.shape, array[ii,101,:],array[ii,102,:]
983
1004
984 # fig = plt.figure(figsize=(6,5))
1005 # fig = plt.figure(figsize=(6,5))
985 # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
1006 # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
986 # ax = fig.add_axes([left, bottom, width, height])
1007 # ax = fig.add_axes([left, bottom, width, height])
987 # x = range(num_prof)
1008 # x = range(num_prof)
988 # y = range(num_hei)
1009 # y = range(num_hei)
989 # cp = ax.contour(y,x,tmp)
1010 # cp = ax.contour(y,x,tmp)
990 # ax.clabel(cp, inline=True,fontsize=10)
1011 # ax.clabel(cp, inline=True,fontsize=10)
991 # plt.show()
1012 # plt.show()
992
1013
993 #indxs = WHERE(FINITE(tmp) AND tmp GT 0,cindxs)
1014 #indxs = WHERE(FINITE(tmp) AND tmp GT 0,cindxs)
994 tmp = numpy.reshape(tmp,num_prof*num_hei)
1015 tmp = numpy.reshape(tmp,num_prof*num_hei)
995 indxs1 = (numpy.isfinite(tmp)==True).nonzero()
1016 indxs1 = (numpy.isfinite(tmp)==True).nonzero()
996 indxs2 = (tmp > 0).nonzero()
1017 indxs2 = (tmp > 0).nonzero()
997
1018
998 indxs1 = (indxs1[0])
1019 indxs1 = (indxs1[0])
999 indxs2 = indxs2[0]
1020 indxs2 = indxs2[0]
1000 #indxs1 = numpy.array(indxs1[0])
1021 #indxs1 = numpy.array(indxs1[0])
1001 #indxs2 = numpy.array(indxs2[0])
1022 #indxs2 = numpy.array(indxs2[0])
1002 indxs = None
1023 indxs = None
1003 #print indxs1 , indxs2
1024 #print indxs1 , indxs2
1004 for iv in range(len(indxs2)):
1025 for iv in range(len(indxs2)):
1005 indv = numpy.array((indxs1 == indxs2[iv]).nonzero())
1026 indv = numpy.array((indxs1 == indxs2[iv]).nonzero())
1006 #print len(indxs2), indv
1027 #print len(indxs2), indv
1007 if len(indv[0]) > 0 :
1028 if len(indv[0]) > 0 :
1008 indxs = numpy.concatenate((indxs,indxs2[iv]), axis=None)
1029 indxs = numpy.concatenate((indxs,indxs2[iv]), axis=None)
1009 # print indxs
1030 # print indxs
1010 indxs = indxs[1:]
1031 indxs = indxs[1:]
1011 #print(indxs, len(indxs))
1032 #print(indxs, len(indxs))
1012 if len(indxs) < 4 :
1033 if len(indxs) < 4 :
1013 array[ii,:,:] = 0.
1034 array[ii,:,:] = 0.
1014 return
1035 return
1015
1036
1016 xpos = numpy.mod(indxs ,num_hei)
1037 xpos = numpy.mod(indxs ,num_hei)
1017 ypos = (indxs / num_hei)
1038 ypos = (indxs / num_hei)
1018 sx = numpy.argsort(xpos) # Ordering respect to "x" (time)
1039 sx = numpy.argsort(xpos) # Ordering respect to "x" (time)
1019 #print sx
1040 #print sx
1020 xpos = xpos[sx]
1041 xpos = xpos[sx]
1021 ypos = ypos[sx]
1042 ypos = ypos[sx]
1022
1043
1023 # *********************************** Cleaning isolated points **********************************
1044 # *********************************** Cleaning isolated points **********************************
1024 ic = 0
1045 ic = 0
1025 while True :
1046 while True :
1026 r = numpy.sqrt(list(numpy.power((xpos[ic]-xpos),2)+ numpy.power((ypos[ic]-ypos),2)))
1047 r = numpy.sqrt(list(numpy.power((xpos[ic]-xpos),2)+ numpy.power((ypos[ic]-ypos),2)))
1027 #no_coh = WHERE(FINITE(r) AND (r LE rth),cno_coh)
1048 #no_coh = WHERE(FINITE(r) AND (r LE rth),cno_coh)
1028 #plt.plot(r)
1049 #plt.plot(r)
1029 #plt.show()
1050 #plt.show()
1030 no_coh1 = (numpy.isfinite(r)==True).nonzero()
1051 no_coh1 = (numpy.isfinite(r)==True).nonzero()
1031 no_coh2 = (r <= rth).nonzero()
1052 no_coh2 = (r <= rth).nonzero()
1032 #print r, no_coh1, no_coh2
1053 #print r, no_coh1, no_coh2
1033 no_coh1 = numpy.array(no_coh1[0])
1054 no_coh1 = numpy.array(no_coh1[0])
1034 no_coh2 = numpy.array(no_coh2[0])
1055 no_coh2 = numpy.array(no_coh2[0])
1035 no_coh = None
1056 no_coh = None
1036 #print valid1 , valid2
1057 #print valid1 , valid2
1037 for iv in range(len(no_coh2)):
1058 for iv in range(len(no_coh2)):
1038 indv = numpy.array((no_coh1 == no_coh2[iv]).nonzero())
1059 indv = numpy.array((no_coh1 == no_coh2[iv]).nonzero())
1039 if len(indv[0]) > 0 :
1060 if len(indv[0]) > 0 :
1040 no_coh = numpy.concatenate((no_coh,no_coh2[iv]), axis=None)
1061 no_coh = numpy.concatenate((no_coh,no_coh2[iv]), axis=None)
1041 no_coh = no_coh[1:]
1062 no_coh = no_coh[1:]
1042 #print len(no_coh), no_coh
1063 #print len(no_coh), no_coh
1043 if len(no_coh) < 4 :
1064 if len(no_coh) < 4 :
1044 #print xpos[ic], ypos[ic], ic
1065 #print xpos[ic], ypos[ic], ic
1045 # plt.plot(r)
1066 # plt.plot(r)
1046 # plt.show()
1067 # plt.show()
1047 xpos[ic] = numpy.nan
1068 xpos[ic] = numpy.nan
1048 ypos[ic] = numpy.nan
1069 ypos[ic] = numpy.nan
1049
1070
1050 ic = ic + 1
1071 ic = ic + 1
1051 if (ic == len(indxs)) :
1072 if (ic == len(indxs)) :
1052 break
1073 break
1053 #print( xpos, ypos)
1074 #print( xpos, ypos)
1054
1075
1055 indxs = (numpy.isfinite(list(xpos))==True).nonzero()
1076 indxs = (numpy.isfinite(list(xpos))==True).nonzero()
1056 #print indxs[0]
1077 #print indxs[0]
1057 if len(indxs[0]) < 4 :
1078 if len(indxs[0]) < 4 :
1058 array[ii,:,:] = 0.
1079 array[ii,:,:] = 0.
1059 return
1080 return
1060
1081
1061 xpos = xpos[indxs[0]]
1082 xpos = xpos[indxs[0]]
1062 ypos = ypos[indxs[0]]
1083 ypos = ypos[indxs[0]]
1063 for i in range(0,len(ypos)):
1084 for i in range(0,len(ypos)):
1064 ypos[i]=int(ypos[i])
1085 ypos[i]=int(ypos[i])
1065 junk = tmp
1086 junk = tmp
1066 tmp = junk*0.0
1087 tmp = junk*0.0
1067
1088
1068 tmp[list(xpos + (ypos*num_hei))] = junk[list(xpos + (ypos*num_hei))]
1089 tmp[list(xpos + (ypos*num_hei))] = junk[list(xpos + (ypos*num_hei))]
1069 array[ii,:,:] = numpy.reshape(tmp,(num_prof,num_hei))
1090 array[ii,:,:] = numpy.reshape(tmp,(num_prof,num_hei))
1070
1091
1071 #print array.shape
1092 #print array.shape
1072 #tmp = numpy.reshape(tmp,(num_prof,num_hei))
1093 #tmp = numpy.reshape(tmp,(num_prof,num_hei))
1073 #print tmp.shape
1094 #print tmp.shape
1074
1095
1075 # fig = plt.figure(figsize=(6,5))
1096 # fig = plt.figure(figsize=(6,5))
1076 # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
1097 # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
1077 # ax = fig.add_axes([left, bottom, width, height])
1098 # ax = fig.add_axes([left, bottom, width, height])
1078 # x = range(num_prof)
1099 # x = range(num_prof)
1079 # y = range(num_hei)
1100 # y = range(num_hei)
1080 # cp = ax.contour(y,x,array[ii,:,:])
1101 # cp = ax.contour(y,x,array[ii,:,:])
1081 # ax.clabel(cp, inline=True,fontsize=10)
1102 # ax.clabel(cp, inline=True,fontsize=10)
1082 # plt.show()
1103 # plt.show()
1083 return array
1104 return array
1084
1105
1085
1106
1086 class IntegrationFaradaySpectra(Operation):
1107 class IntegrationFaradaySpectra(Operation):
1087
1108
1088 __profIndex = 0
1109 __profIndex = 0
1089 __withOverapping = False
1110 __withOverapping = False
1090
1111
1091 __byTime = False
1112 __byTime = False
1092 __initime = None
1113 __initime = None
1093 __lastdatatime = None
1114 __lastdatatime = None
1094 __integrationtime = None
1115 __integrationtime = None
1095
1116
1096 __buffer_spc = None
1117 __buffer_spc = None
1097 __buffer_cspc = None
1118 __buffer_cspc = None
1098 __buffer_dc = None
1119 __buffer_dc = None
1099
1120
1100 __dataReady = False
1121 __dataReady = False
1101
1122
1102 __timeInterval = None
1123 __timeInterval = None
1103
1124 n_ints = None #matriz de numero de integracions (CH,HEI)
1104 n = None
1125 n = None
1105 minHei_ind = None
1126 minHei_ind = None
1106 maxHei_ind = None
1127 maxHei_ind = None
1107 navg = 1.0
1128 navg = 1.0
1108 factor = 0.0
1129 factor = 0.0
1130 dataoutliers = None # (CHANNELS, HEIGHTS)
1109
1131
1110 def __init__(self):
1132 def __init__(self):
1111
1133
1112 Operation.__init__(self)
1134 Operation.__init__(self)
1113
1135
1114 def setup(self, dataOut,n=None, timeInterval=None, overlapping=False, DPL=None, minHei=None, maxHei=None, avg=1,factor=0.75):
1136 def setup(self, dataOut,n=None, timeInterval=None, overlapping=False, DPL=None, minHei=None, maxHei=None, avg=1,factor=0.75):
1115 """
1137 """
1116 Set the parameters of the integration class.
1138 Set the parameters of the integration class.
1117
1139
1118 Inputs:
1140 Inputs:
1119
1141
1120 n : Number of coherent integrations
1142 n : Number of coherent integrations
1121 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1143 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1122 overlapping :
1144 overlapping :
1123
1145
1124 """
1146 """
1125
1147
1126 self.__initime = None
1148 self.__initime = None
1127 self.__lastdatatime = 0
1149 self.__lastdatatime = 0
1128
1150
1129 self.__buffer_spc = []
1151 self.__buffer_spc = []
1130 self.__buffer_cspc = []
1152 self.__buffer_cspc = []
1131 self.__buffer_dc = 0
1153 self.__buffer_dc = 0
1132
1154
1133 self.__profIndex = 0
1155 self.__profIndex = 0
1134 self.__dataReady = False
1156 self.__dataReady = False
1135 self.__byTime = False
1157 self.__byTime = False
1136
1158
1137 self.factor = factor
1159 self.factor = factor
1138 self.navg = avg
1160 self.navg = avg
1139 #self.ByLags = dataOut.ByLags ###REDEFINIR
1161 #self.ByLags = dataOut.ByLags ###REDEFINIR
1140 self.ByLags = False
1162 self.ByLags = False
1163 self.maxProfilesInt = 1
1141
1164
1142 if DPL != None:
1165 if DPL != None:
1143 self.DPL=DPL
1166 self.DPL=DPL
1144 else:
1167 else:
1145 #self.DPL=dataOut.DPL ###REDEFINIR
1168 #self.DPL=dataOut.DPL ###REDEFINIR
1146 self.DPL=0
1169 self.DPL=0
1147
1170
1148 if n is None and timeInterval is None:
1171 if n is None and timeInterval is None:
1149 raise ValueError("n or timeInterval should be specified ...")
1172 raise ValueError("n or timeInterval should be specified ...")
1150
1173
1151 if n is not None:
1174 if n is not None:
1152 self.n = int(n)
1175 self.n = int(n)
1153 else:
1176 else:
1154 self.__integrationtime = int(timeInterval)
1177 self.__integrationtime = int(timeInterval)
1155 self.n = None
1178 self.n = None
1156 self.__byTime = True
1179 self.__byTime = True
1157
1180
1158 if minHei == None:
1181 if minHei == None:
1159 minHei = self.dataOut.heightList[0]
1182 minHei = self.dataOut.heightList[0]
1160
1183
1161 if maxHei == None:
1184 if maxHei == None:
1162 maxHei = self.dataOut.heightList[-1]
1185 maxHei = self.dataOut.heightList[-1]
1163
1186
1164 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
1187 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
1165 print('minHei: %.2f is out of the heights range' % (minHei))
1188 print('minHei: %.2f is out of the heights range' % (minHei))
1166 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
1189 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
1167 minHei = self.dataOut.heightList[0]
1190 minHei = self.dataOut.heightList[0]
1168
1191
1169 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
1192 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
1170 print('maxHei: %.2f is out of the heights range' % (maxHei))
1193 print('maxHei: %.2f is out of the heights range' % (maxHei))
1171 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
1194 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
1172 maxHei = self.dataOut.heightList[-1]
1195 maxHei = self.dataOut.heightList[-1]
1173
1196
1174 ind_list1 = numpy.where(self.dataOut.heightList >= minHei)
1197 ind_list1 = numpy.where(self.dataOut.heightList >= minHei)
1175 ind_list2 = numpy.where(self.dataOut.heightList <= maxHei)
1198 ind_list2 = numpy.where(self.dataOut.heightList <= maxHei)
1176 self.minHei_ind = ind_list1[0][0]
1199 self.minHei_ind = ind_list1[0][0]
1177 self.maxHei_ind = ind_list2[0][-1]
1200 self.maxHei_ind = ind_list2[0][-1]
1178
1201
1179 def putData(self, data_spc, data_cspc, data_dc):
1202 def putData(self, data_spc, data_cspc, data_dc):
1180 """
1203 """
1181 Add a profile to the __buffer_spc and increase in one the __profileIndex
1204 Add a profile to the __buffer_spc and increase in one the __profileIndex
1182
1205
1183 """
1206 """
1184
1207
1185 self.__buffer_spc.append(data_spc)
1208 self.__buffer_spc.append(data_spc)
1186
1209
1187 if data_cspc is None:
1210 if data_cspc is None:
1188 self.__buffer_cspc = None
1211 self.__buffer_cspc = None
1189 else:
1212 else:
1190 self.__buffer_cspc.append(data_cspc)
1213 self.__buffer_cspc.append(data_cspc)
1191
1214
1192 if data_dc is None:
1215 if data_dc is None:
1193 self.__buffer_dc = None
1216 self.__buffer_dc = None
1194 else:
1217 else:
1195 self.__buffer_dc += data_dc
1218 self.__buffer_dc += data_dc
1196
1219
1197 self.__profIndex += 1
1220 self.__profIndex += 1
1198
1221
1199 return
1222 return
1200
1223
1201 def hildebrand_sekhon_Integration(self,sortdata,navg, factor):
1224 def hildebrand_sekhon_Integration(self,sortdata,navg, factor):
1202 #data debe estar ordenado
1225 #data debe estar ordenado
1203 #sortdata = numpy.sort(data, axis=None)
1226 #sortdata = numpy.sort(data, axis=None)
1204 #sortID=data.argsort()
1227 #sortID=data.argsort()
1205 lenOfData = len(sortdata)
1228 lenOfData = len(sortdata)
1206 nums_min = lenOfData*factor
1229 nums_min = lenOfData*factor
1207 if nums_min <= 5:
1230 if nums_min <= 5:
1208 nums_min = 5
1231 nums_min = 5
1209 sump = 0.
1232 sump = 0.
1210 sumq = 0.
1233 sumq = 0.
1211 j = 0
1234 j = 0
1212 cont = 1
1235 cont = 1
1213 while((cont == 1)and(j < lenOfData)):
1236 while((cont == 1)and(j < lenOfData)):
1214 sump += sortdata[j]
1237 sump += sortdata[j]
1215 sumq += sortdata[j]**2
1238 sumq += sortdata[j]**2
1216 if j > nums_min:
1239 if j > nums_min:
1217 rtest = float(j)/(j-1) + 1.0/navg
1240 rtest = float(j)/(j-1) + 1.0/navg
1218 if ((sumq*j) > (rtest*sump**2)):
1241 if ((sumq*j) > (rtest*sump**2)):
1219 j = j - 1
1242 j = j - 1
1220 sump = sump - sortdata[j]
1243 sump = sump - sortdata[j]
1221 sumq = sumq - sortdata[j]**2
1244 sumq = sumq - sortdata[j]**2
1222 cont = 0
1245 cont = 0
1223 j += 1
1246 j += 1
1224 #lnoise = sump / j
1247 #lnoise = sump / j
1225 #print("H S done")
1248 #print("H S done")
1226 #return j,sortID
1249 #return j,sortID
1227 return j
1250 return j
1228
1251
1229
1252
1230 def pushData(self):
1253 def pushData(self):
1231 """
1254 """
1232 Return the sum of the last profiles and the profiles used in the sum.
1255 Return the sum of the last profiles and the profiles used in the sum.
1233
1256
1234 Affected:
1257 Affected:
1235
1258
1236 self.__profileIndex
1259 self.__profileIndex
1237
1260
1238 """
1261 """
1239 bufferH=None
1262 bufferH=None
1240 buffer=None
1263 buffer=None
1241 buffer1=None
1264 buffer1=None
1242 buffer_cspc=None
1265 buffer_cspc=None
1243 self.__buffer_spc=numpy.array(self.__buffer_spc)
1266 self.__buffer_spc=numpy.array(self.__buffer_spc)
1244 try:
1267 try:
1245 self.__buffer_cspc=numpy.array(self.__buffer_cspc)
1268 self.__buffer_cspc=numpy.array(self.__buffer_cspc)
1246 except :
1269 except :
1247 #print("No cpsc",e)
1270 #print("No cpsc",e)
1248 pass
1271 pass
1249 #print("FREQ_DC",self.__buffer_spc.shape,self.__buffer_cspc.shape)
1272 #print("FREQ_DC",self.__buffer_spc.shape,self.__buffer_cspc.shape)
1250
1273
1251 freq_dc = int(self.__buffer_spc.shape[2] / 2)
1274 freq_dc = int(self.__buffer_spc.shape[2] / 2)
1252 #print("FREQ_DC",freq_dc,self.__buffer_spc.shape,self.nHeights)
1275 #print("FREQ_DC",freq_dc,self.__buffer_spc.shape,self.nHeights)
1253
1276
1277 self.dataOutliers = numpy.zeros((self.nChannels,self.nHeights)) # --> almacen de outliers
1278
1254 for k in range(self.minHei_ind,self.maxHei_ind):
1279 for k in range(self.minHei_ind,self.maxHei_ind):
1255 try:
1280 try:
1256 buffer_cspc=numpy.copy(self.__buffer_cspc[:,:,:,k])
1281 buffer_cspc=numpy.copy(self.__buffer_cspc[:,:,:,k])
1257 except:
1282 except:
1258 #print("No cpsc",e)
1283 #print("No cpsc",e)
1259 pass
1284 pass
1260 outliers_IDs_cspc=[]
1285 outliers_IDs_cspc=[]
1261 cspc_outliers_exist=False
1286 cspc_outliers_exist=False
1262 for i in range(self.nChannels):#dataOut.nChannels):
1287 for i in range(self.nChannels):#dataOut.nChannels):
1263
1288
1264 buffer1=numpy.copy(self.__buffer_spc[:,i,:,k])
1289 buffer1=numpy.copy(self.__buffer_spc[:,i,:,k])
1265 indexes=[]
1290 indexes=[]
1266 #sortIDs=[]
1291 #sortIDs=[]
1267 outliers_IDs=[]
1292 outliers_IDs=[]
1268
1293
1269 for j in range(self.nProfiles):
1294 for j in range(self.nProfiles): #frecuencias en el tiempo
1270 # if i==0 and j==freq_dc: #NOT CONSIDERING DC PROFILE AT CHANNEL 0
1295 # if i==0 and j==freq_dc: #NOT CONSIDERING DC PROFILE AT CHANNEL 0
1271 # continue
1296 # continue
1272 # if i==1 and j==0: #NOT CONSIDERING DC PROFILE AT CHANNEL 1
1297 # if i==1 and j==0: #NOT CONSIDERING DC PROFILE AT CHANNEL 1
1273 # continue
1298 # continue
1274 buffer=buffer1[:,j]
1299 buffer=buffer1[:,j]
1275 sortdata = numpy.sort(buffer, axis=None)
1300 sortdata = numpy.sort(buffer, axis=None)
1301
1276 sortID=buffer.argsort()
1302 sortID=buffer.argsort()
1277 index = _noise.hildebrand_sekhon2(sortdata,self.navg)
1303 index = _noise.hildebrand_sekhon2(sortdata,self.navg)
1278
1304
1279 #index,sortID=self.hildebrand_sekhon_Integration(buffer,1,self.factor)
1305 #index,sortID=self.hildebrand_sekhon_Integration(buffer,1,self.factor)
1280
1306
1307 # fig,ax = plt.subplots()
1308 # ax.set_title(str(k)+" "+str(j))
1309 # x=range(len(sortdata))
1310 # ax.scatter(x,sortdata)
1311 # ax.axvline(index)
1312 # plt.show()
1313
1281 indexes.append(index)
1314 indexes.append(index)
1282 #sortIDs.append(sortID)
1315 #sortIDs.append(sortID)
1283 outliers_IDs=numpy.append(outliers_IDs,sortID[index:])
1316 outliers_IDs=numpy.append(outliers_IDs,sortID[index:])
1284
1317
1318 #print("Outliers: ",outliers_IDs)
1285 outliers_IDs=numpy.array(outliers_IDs)
1319 outliers_IDs=numpy.array(outliers_IDs)
1286 outliers_IDs=outliers_IDs.ravel()
1320 outliers_IDs=outliers_IDs.ravel()
1287 outliers_IDs=numpy.unique(outliers_IDs)
1321 outliers_IDs=numpy.unique(outliers_IDs)
1288 outliers_IDs=outliers_IDs.astype(numpy.dtype('int64'))
1322 outliers_IDs=outliers_IDs.astype(numpy.dtype('int64'))
1289 indexes=numpy.array(indexes)
1323 indexes=numpy.array(indexes)
1290 indexmin=numpy.min(indexes)
1324 indexmin=numpy.min(indexes)
1291
1325
1326
1327 #print(indexmin,buffer1.shape[0], k)
1328
1329 # fig,ax = plt.subplots()
1330 # ax.plot(sortdata)
1331 # ax2 = ax.twinx()
1332 # x=range(len(indexes))
1333 # #plt.scatter(x,indexes)
1334 # ax2.scatter(x,indexes)
1335 # plt.show()
1336
1292 if indexmin != buffer1.shape[0]:
1337 if indexmin != buffer1.shape[0]:
1293 if self.nChannels > 1:
1338 if self.nChannels > 1:
1294 cspc_outliers_exist= True
1339 cspc_outliers_exist= True
1295
1340
1296 lt=outliers_IDs
1341 lt=outliers_IDs
1297 avg=numpy.mean(buffer1[[t for t in range(buffer1.shape[0]) if t not in lt],:],axis=0)
1342 #avg=numpy.mean(buffer1[[t for t in range(buffer1.shape[0]) if t not in lt],:],axis=0)
1298
1343
1299 for p in list(outliers_IDs):
1344 for p in list(outliers_IDs):
1300 buffer1[p,:]=avg
1345 #buffer1[p,:]=avg
1346 buffer1[p,:] = numpy.NaN
1347
1348 self.dataOutliers[i,k] = len(outliers_IDs)
1301
1349
1302 self.__buffer_spc[:,i,:,k]=numpy.copy(buffer1)
1350 self.__buffer_spc[:,i,:,k]=numpy.copy(buffer1)
1303 ###cspc IDs
1351
1304 #indexmin_cspc+=indexmin_cspc
1305 outliers_IDs_cspc=numpy.append(outliers_IDs_cspc,outliers_IDs)
1352 outliers_IDs_cspc=numpy.append(outliers_IDs_cspc,outliers_IDs)
1306
1353
1307 #if not breakFlag:
1354
1355
1308 outliers_IDs_cspc=outliers_IDs_cspc.astype(numpy.dtype('int64'))
1356 outliers_IDs_cspc=outliers_IDs_cspc.astype(numpy.dtype('int64'))
1309 if cspc_outliers_exist :
1357 if cspc_outliers_exist :
1310 #sortdata=numpy.sort(buffer_cspc,axis=0)
1358
1311 #avg=numpy.mean(sortdata[:indexmin_cpsc,:],axis=0)
1312 lt=outliers_IDs_cspc
1359 lt=outliers_IDs_cspc
1313
1360
1314 avg=numpy.mean(buffer_cspc[[t for t in range(buffer_cspc.shape[0]) if t not in lt],:],axis=0)
1361 #avg=numpy.mean(buffer_cspc[[t for t in range(buffer_cspc.shape[0]) if t not in lt],:],axis=0)
1315 for p in list(outliers_IDs_cspc):
1362 for p in list(outliers_IDs_cspc):
1316 buffer_cspc[p,:]=avg
1363 #buffer_cspc[p,:]=avg
1364 buffer_cspc[p,:] = numpy.NaN
1317
1365
1318 try:
1366 try:
1319 self.__buffer_cspc[:,:,:,k]=numpy.copy(buffer_cspc)
1367 self.__buffer_cspc[:,:,:,k]=numpy.copy(buffer_cspc)
1320 except:
1368 except:
1321 #print("No cpsc",e)
1369 #print("No cpsc",e)
1322 pass
1370 pass
1323 #else:
1371 #else:
1324 #break
1372 #break
1325
1373
1326
1374
1327
1375
1328
1376 nOutliers = len(outliers_IDs)
1377 #print("Outliers n: ",self.dataOutliers,nOutliers)
1329 buffer=None
1378 buffer=None
1330 bufferH=None
1379 bufferH=None
1331 buffer1=None
1380 buffer1=None
1332 buffer_cspc=None
1381 buffer_cspc=None
1333
1382
1334 #print("cpsc",self.__buffer_cspc[:,0,0,0,0])
1335 #exit()
1336
1383
1337 buffer=None
1384 buffer=None
1338 #print(self.__buffer_spc[:,1,3,20,0])
1385
1339 #print(self.__buffer_spc[:,1,5,37,0])
1386 #data_spc = numpy.sum(self.__buffer_spc,axis=0)
1340 data_spc = numpy.sum(self.__buffer_spc,axis=0)
1387 data_spc = numpy.nansum(self.__buffer_spc,axis=0)
1341 try:
1388 try:
1342 data_cspc = numpy.sum(self.__buffer_cspc,axis=0)
1389 #data_cspc = numpy.sum(self.__buffer_cspc,axis=0)
1390 data_cspc = numpy.nansum(self.__buffer_cspc,axis=0)
1343 except:
1391 except:
1344 #print("No cpsc",e)
1392 #print("No cpsc",e)
1345 pass
1393 pass
1346
1394
1347
1395
1348 #print(numpy.shape(data_spc))
1349 #data_spc[1,4,20,0]=numpy.nan
1350
1351 #data_cspc = self.__buffer_cspc
1352 #print("pushData pre Done")
1353 data_dc = self.__buffer_dc
1396 data_dc = self.__buffer_dc
1354 n = self.__profIndex
1397 #(CH, HEIGH)
1398 self.maxProfilesInt = self.__profIndex
1399 n = self.__profIndex - self.dataOutliers
1355
1400
1356 self.__buffer_spc = []
1401 self.__buffer_spc = []
1357 self.__buffer_cspc = []
1402 self.__buffer_cspc = []
1358 self.__buffer_dc = 0
1403 self.__buffer_dc = 0
1359 self.__profIndex = 0
1404 self.__profIndex = 0
1360 #print("pushData Done")
1405
1361 return data_spc, data_cspc, data_dc, n
1406 return data_spc, data_cspc, data_dc, n
1362
1407
1363 def byProfiles(self, *args):
1408 def byProfiles(self, *args):
1364
1409
1365 self.__dataReady = False
1410 self.__dataReady = False
1366 avgdata_spc = None
1411 avgdata_spc = None
1367 avgdata_cspc = None
1412 avgdata_cspc = None
1368 avgdata_dc = None
1413 avgdata_dc = None
1369
1414
1370 self.putData(*args)
1415 self.putData(*args)
1371
1416
1372 if self.__profIndex == self.n:
1417 if self.__profIndex == self.n:
1373
1418
1374 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1419 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1375 self.n = n
1420 self.n_ints = n
1376 self.__dataReady = True
1421 self.__dataReady = True
1377
1422
1378 return avgdata_spc, avgdata_cspc, avgdata_dc
1423 return avgdata_spc, avgdata_cspc, avgdata_dc
1379
1424
1380 def byTime(self, datatime, *args):
1425 def byTime(self, datatime, *args):
1381
1426
1382 self.__dataReady = False
1427 self.__dataReady = False
1383 avgdata_spc = None
1428 avgdata_spc = None
1384 avgdata_cspc = None
1429 avgdata_cspc = None
1385 avgdata_dc = None
1430 avgdata_dc = None
1386
1431
1387 self.putData(*args)
1432 self.putData(*args)
1388
1433
1389 if (datatime - self.__initime) >= self.__integrationtime:
1434 if (datatime - self.__initime) >= self.__integrationtime:
1390 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1435 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1391 self.n = n
1436 self.n_ints = n
1392 self.__dataReady = True
1437 self.__dataReady = True
1393
1438
1394 return avgdata_spc, avgdata_cspc, avgdata_dc
1439 return avgdata_spc, avgdata_cspc, avgdata_dc
1395
1440
1396 def integrate(self, datatime, *args):
1441 def integrate(self, datatime, *args):
1397
1442
1398 if self.__profIndex == 0:
1443 if self.__profIndex == 0:
1399 self.__initime = datatime
1444 self.__initime = datatime
1400
1445
1401 if self.__byTime:
1446 if self.__byTime:
1402 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
1447 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
1403 datatime, *args)
1448 datatime, *args)
1404 else:
1449 else:
1405 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1450 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1406
1451
1407 if not self.__dataReady:
1452 if not self.__dataReady:
1408 return None, None, None, None
1453 return None, None, None, None
1409
1454
1410 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
1455 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
1411
1456
1412 def run(self, dataOut, n=None, DPL = None,timeInterval=None, overlapping=False, minHei=None, maxHei=None, avg=1, factor=0.75):
1457 def run(self, dataOut, n=None, DPL = None,timeInterval=None, overlapping=False, minHei=None, maxHei=None, avg=1, factor=0.75):
1413 self.dataOut = dataOut.copy()
1458 self.dataOut = dataOut.copy()
1414 if n == 1:
1459 if n == 1:
1415 return self.dataOut
1460 return self.dataOut
1416
1461
1417 self.dataOut.flagNoData = True
1462 self.dataOut.flagNoData = True
1418 if self.dataOut.nChannels == 1:
1463 if self.dataOut.nChannels == 1:
1419 self.dataOut.data_cspc = None #si es un solo canal no vale la pena acumular DATOS
1464 self.dataOut.data_cspc = None #si es un solo canal no vale la pena acumular DATOS
1420 #print(self.dataOut.data_spc.shape, self.dataOut.data_cspc)
1465 #print(self.dataOut.data_spc.shape, self.dataOut.data_cspc)
1421 if not self.isConfig:
1466 if not self.isConfig:
1422 self.setup(self.dataOut, n, timeInterval, overlapping,DPL ,minHei, maxHei, avg, factor)
1467 self.setup(self.dataOut, n, timeInterval, overlapping,DPL ,minHei, maxHei, avg, factor)
1423 self.isConfig = True
1468 self.isConfig = True
1424
1469
1425 if not self.ByLags:
1470 if not self.ByLags:
1426 self.nProfiles=self.dataOut.nProfiles
1471 self.nProfiles=self.dataOut.nProfiles
1427 self.nChannels=self.dataOut.nChannels
1472 self.nChannels=self.dataOut.nChannels
1428 self.nHeights=self.dataOut.nHeights
1473 self.nHeights=self.dataOut.nHeights
1429 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
1474 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
1430 self.dataOut.data_spc,
1475 self.dataOut.data_spc,
1431 self.dataOut.data_cspc,
1476 self.dataOut.data_cspc,
1432 self.dataOut.data_dc)
1477 self.dataOut.data_dc)
1433 else:
1478 else:
1434 self.nProfiles=self.dataOut.nProfiles
1479 self.nProfiles=self.dataOut.nProfiles
1435 self.nChannels=self.dataOut.nChannels
1480 self.nChannels=self.dataOut.nChannels
1436 self.nHeights=self.dataOut.nHeights
1481 self.nHeights=self.dataOut.nHeights
1437 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
1482 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(self.dataOut.utctime,
1438 self.dataOut.dataLag_spc,
1483 self.dataOut.dataLag_spc,
1439 self.dataOut.dataLag_cspc,
1484 self.dataOut.dataLag_cspc,
1440 self.dataOut.dataLag_dc)
1485 self.dataOut.dataLag_dc)
1441
1486
1442 if self.__dataReady:
1487 if self.__dataReady:
1443
1488
1444 if not self.ByLags:
1489 if not self.ByLags:
1445 if self.nChannels == 1:
1490 if self.nChannels == 1:
1446 #print("f int", avgdata_spc.shape)
1491 #print("f int", avgdata_spc.shape)
1447 self.dataOut.data_spc = avgdata_spc
1492 self.dataOut.data_spc = avgdata_spc
1448 self.dataOut.data_cspc = avgdata_spc
1493 self.dataOut.data_cspc = avgdata_spc
1449 else:
1494 else:
1450 self.dataOut.data_spc = numpy.squeeze(avgdata_spc)
1495 self.dataOut.data_spc = numpy.squeeze(avgdata_spc)
1451 self.dataOut.data_cspc = numpy.squeeze(avgdata_cspc)
1496 self.dataOut.data_cspc = numpy.squeeze(avgdata_cspc)
1452 self.dataOut.data_dc = avgdata_dc
1497 self.dataOut.data_dc = avgdata_dc
1453
1498 self.dataOut.data_outlier = self.dataOutliers
1454
1499
1455 else:
1500 else:
1456 self.dataOut.dataLag_spc = avgdata_spc
1501 self.dataOut.dataLag_spc = avgdata_spc
1457 self.dataOut.dataLag_cspc = avgdata_cspc
1502 self.dataOut.dataLag_cspc = avgdata_cspc
1458 self.dataOut.dataLag_dc = avgdata_dc
1503 self.dataOut.dataLag_dc = avgdata_dc
1459
1504
1460 self.dataOut.data_spc=self.dataOut.dataLag_spc[:,:,:,self.dataOut.LagPlot]
1505 self.dataOut.data_spc=self.dataOut.dataLag_spc[:,:,:,self.dataOut.LagPlot]
1461 self.dataOut.data_cspc=self.dataOut.dataLag_cspc[:,:,:,self.dataOut.LagPlot]
1506 self.dataOut.data_cspc=self.dataOut.dataLag_cspc[:,:,:,self.dataOut.LagPlot]
1462 self.dataOut.data_dc=self.dataOut.dataLag_dc[:,:,self.dataOut.LagPlot]
1507 self.dataOut.data_dc=self.dataOut.dataLag_dc[:,:,self.dataOut.LagPlot]
1463
1508
1464
1509
1465 self.dataOut.nIncohInt *= self.n
1510 self.dataOut.nIncohInt *= self.n_ints
1511 self.dataOut.max_nIncohInt = self.maxProfilesInt
1512 #print(self.dataOut.max_nIncohInt)
1466 self.dataOut.utctime = avgdatatime
1513 self.dataOut.utctime = avgdatatime
1467 self.dataOut.flagNoData = False
1514 self.dataOut.flagNoData = False
1468
1515 #print("Faraday Integration DONE...")
1469 return self.dataOut
1516 return self.dataOut
1470
1517
1471 class removeInterference(Operation):
1518 class removeInterference(Operation):
1472
1519
1473 def removeInterference2(self):
1520 def removeInterference2(self):
1474
1521
1475 cspc = self.dataOut.data_cspc
1522 cspc = self.dataOut.data_cspc
1476 spc = self.dataOut.data_spc
1523 spc = self.dataOut.data_spc
1477 Heights = numpy.arange(cspc.shape[2])
1524 Heights = numpy.arange(cspc.shape[2])
1478 realCspc = numpy.abs(cspc)
1525 realCspc = numpy.abs(cspc)
1479
1526
1480 for i in range(cspc.shape[0]):
1527 for i in range(cspc.shape[0]):
1481 LinePower= numpy.sum(realCspc[i], axis=0)
1528 LinePower= numpy.sum(realCspc[i], axis=0)
1482 Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
1529 Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
1483 SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ]
1530 SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ]
1484 InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
1531 InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
1485 InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
1532 InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
1486 InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
1533 InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
1487
1534
1488
1535
1489 InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
1536 InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
1490 #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
1537 #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
1491 if len(InterferenceRange)<int(cspc.shape[1]*0.3):
1538 if len(InterferenceRange)<int(cspc.shape[1]*0.3):
1492 cspc[i,InterferenceRange,:] = numpy.NaN
1539 cspc[i,InterferenceRange,:] = numpy.NaN
1493
1540
1494 self.dataOut.data_cspc = cspc
1541 self.dataOut.data_cspc = cspc
1495
1542
1496 def removeInterference(self, interf = 2, hei_interf = None, nhei_interf = None, offhei_interf = None):
1543 def removeInterference(self, interf = 2, hei_interf = None, nhei_interf = None, offhei_interf = None):
1497
1544
1498 jspectra = self.dataOut.data_spc
1545 jspectra = self.dataOut.data_spc
1499 jcspectra = self.dataOut.data_cspc
1546 jcspectra = self.dataOut.data_cspc
1500 jnoise = self.dataOut.getNoise()
1547 jnoise = self.dataOut.getNoise()
1501 num_incoh = self.dataOut.nIncohInt
1548 num_incoh = self.dataOut.nIncohInt
1502
1549
1503 num_channel = jspectra.shape[0]
1550 num_channel = jspectra.shape[0]
1504 num_prof = jspectra.shape[1]
1551 num_prof = jspectra.shape[1]
1505 num_hei = jspectra.shape[2]
1552 num_hei = jspectra.shape[2]
1506
1553
1507 # hei_interf
1554 # hei_interf
1508 if hei_interf is None:
1555 if hei_interf is None:
1509 count_hei = int(num_hei / 2)
1556 count_hei = int(num_hei / 2)
1510 hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
1557 hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
1511 hei_interf = numpy.asarray(hei_interf)[0]
1558 hei_interf = numpy.asarray(hei_interf)[0]
1512 # nhei_interf
1559 # nhei_interf
1513 if (nhei_interf == None):
1560 if (nhei_interf == None):
1514 nhei_interf = 5
1561 nhei_interf = 5
1515 if (nhei_interf < 1):
1562 if (nhei_interf < 1):
1516 nhei_interf = 1
1563 nhei_interf = 1
1517 if (nhei_interf > count_hei):
1564 if (nhei_interf > count_hei):
1518 nhei_interf = count_hei
1565 nhei_interf = count_hei
1519 if (offhei_interf == None):
1566 if (offhei_interf == None):
1520 offhei_interf = 0
1567 offhei_interf = 0
1521
1568
1522 ind_hei = list(range(num_hei))
1569 ind_hei = list(range(num_hei))
1523 # mask_prof = numpy.asarray(range(num_prof - 2)) + 1
1570 # mask_prof = numpy.asarray(range(num_prof - 2)) + 1
1524 # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
1571 # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
1525 mask_prof = numpy.asarray(list(range(num_prof)))
1572 mask_prof = numpy.asarray(list(range(num_prof)))
1526 num_mask_prof = mask_prof.size
1573 num_mask_prof = mask_prof.size
1527 comp_mask_prof = [0, num_prof / 2]
1574 comp_mask_prof = [0, num_prof / 2]
1528
1575
1529 # noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
1576 # noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
1530 if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
1577 if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
1531 jnoise = numpy.nan
1578 jnoise = numpy.nan
1532 noise_exist = jnoise[0] < numpy.Inf
1579 noise_exist = jnoise[0] < numpy.Inf
1533
1580
1534 # Subrutina de Remocion de la Interferencia
1581 # Subrutina de Remocion de la Interferencia
1535 for ich in range(num_channel):
1582 for ich in range(num_channel):
1536 # Se ordena los espectros segun su potencia (menor a mayor)
1583 # Se ordena los espectros segun su potencia (menor a mayor)
1537 power = jspectra[ich, mask_prof, :]
1584 power = jspectra[ich, mask_prof, :]
1538 power = power[:, hei_interf]
1585 power = power[:, hei_interf]
1539 power = power.sum(axis=0)
1586 power = power.sum(axis=0)
1540 psort = power.ravel().argsort()
1587 psort = power.ravel().argsort()
1541
1588
1542 # Se estima la interferencia promedio en los Espectros de Potencia empleando
1589 # Se estima la interferencia promedio en los Espectros de Potencia empleando
1543 junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
1590 junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
1544 offhei_interf, nhei_interf + offhei_interf))]]]
1591 offhei_interf, nhei_interf + offhei_interf))]]]
1545
1592
1546 if noise_exist:
1593 if noise_exist:
1547 # tmp_noise = jnoise[ich] / num_prof
1594 # tmp_noise = jnoise[ich] / num_prof
1548 tmp_noise = jnoise[ich]
1595 tmp_noise = jnoise[ich]
1549 junkspc_interf = junkspc_interf - tmp_noise
1596 junkspc_interf = junkspc_interf - tmp_noise
1550 #junkspc_interf[:,comp_mask_prof] = 0
1597 #junkspc_interf[:,comp_mask_prof] = 0
1551
1598
1552 jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
1599 jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
1553 jspc_interf = jspc_interf.transpose()
1600 jspc_interf = jspc_interf.transpose()
1554 # Calculando el espectro de interferencia promedio
1601 # Calculando el espectro de interferencia promedio
1555 noiseid = numpy.where(
1602 noiseid = numpy.where(
1556 jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
1603 jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
1557 noiseid = noiseid[0]
1604 noiseid = noiseid[0]
1558 cnoiseid = noiseid.size
1605 cnoiseid = noiseid.size
1559 interfid = numpy.where(
1606 interfid = numpy.where(
1560 jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
1607 jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
1561 interfid = interfid[0]
1608 interfid = interfid[0]
1562 cinterfid = interfid.size
1609 cinterfid = interfid.size
1563
1610
1564 if (cnoiseid > 0):
1611 if (cnoiseid > 0):
1565 jspc_interf[noiseid] = 0
1612 jspc_interf[noiseid] = 0
1566
1613
1567 # Expandiendo los perfiles a limpiar
1614 # Expandiendo los perfiles a limpiar
1568 if (cinterfid > 0):
1615 if (cinterfid > 0):
1569 new_interfid = (
1616 new_interfid = (
1570 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
1617 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
1571 new_interfid = numpy.asarray(new_interfid)
1618 new_interfid = numpy.asarray(new_interfid)
1572 new_interfid = {x for x in new_interfid}
1619 new_interfid = {x for x in new_interfid}
1573 new_interfid = numpy.array(list(new_interfid))
1620 new_interfid = numpy.array(list(new_interfid))
1574 new_cinterfid = new_interfid.size
1621 new_cinterfid = new_interfid.size
1575 else:
1622 else:
1576 new_cinterfid = 0
1623 new_cinterfid = 0
1577
1624
1578 for ip in range(new_cinterfid):
1625 for ip in range(new_cinterfid):
1579 ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
1626 ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
1580 jspc_interf[new_interfid[ip]
1627 jspc_interf[new_interfid[ip]
1581 ] = junkspc_interf[ind[nhei_interf // 2], new_interfid[ip]]
1628 ] = junkspc_interf[ind[nhei_interf // 2], new_interfid[ip]]
1582
1629
1583 jspectra[ich, :, ind_hei] = jspectra[ich, :,
1630 jspectra[ich, :, ind_hei] = jspectra[ich, :,
1584 ind_hei] - jspc_interf # Corregir indices
1631 ind_hei] - jspc_interf # Corregir indices
1585
1632
1586 # Removiendo la interferencia del punto de mayor interferencia
1633 # Removiendo la interferencia del punto de mayor interferencia
1587 ListAux = jspc_interf[mask_prof].tolist()
1634 ListAux = jspc_interf[mask_prof].tolist()
1588 maxid = ListAux.index(max(ListAux))
1635 maxid = ListAux.index(max(ListAux))
1589
1636
1590 if cinterfid > 0:
1637 if cinterfid > 0:
1591 for ip in range(cinterfid * (interf == 2) - 1):
1638 for ip in range(cinterfid * (interf == 2) - 1):
1592 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
1639 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
1593 (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
1640 (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
1594 cind = len(ind)
1641 cind = len(ind)
1595
1642
1596 if (cind > 0):
1643 if (cind > 0):
1597 jspectra[ich, interfid[ip], ind] = tmp_noise * \
1644 jspectra[ich, interfid[ip], ind] = tmp_noise * \
1598 (1 + (numpy.random.uniform(cind) - 0.5) /
1645 (1 + (numpy.random.uniform(cind) - 0.5) /
1599 numpy.sqrt(num_incoh))
1646 numpy.sqrt(num_incoh))
1600
1647
1601 ind = numpy.array([-2, -1, 1, 2])
1648 ind = numpy.array([-2, -1, 1, 2])
1602 xx = numpy.zeros([4, 4])
1649 xx = numpy.zeros([4, 4])
1603
1650
1604 for id1 in range(4):
1651 for id1 in range(4):
1605 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1652 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1606
1653
1607 xx_inv = numpy.linalg.inv(xx)
1654 xx_inv = numpy.linalg.inv(xx)
1608 xx = xx_inv[:, 0]
1655 xx = xx_inv[:, 0]
1609 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1656 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1610 yy = jspectra[ich, mask_prof[ind], :]
1657 yy = jspectra[ich, mask_prof[ind], :]
1611 jspectra[ich, mask_prof[maxid], :] = numpy.dot(
1658 jspectra[ich, mask_prof[maxid], :] = numpy.dot(
1612 yy.transpose(), xx)
1659 yy.transpose(), xx)
1613
1660
1614 indAux = (jspectra[ich, :, :] < tmp_noise *
1661 indAux = (jspectra[ich, :, :] < tmp_noise *
1615 (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
1662 (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
1616 jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
1663 jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
1617 (1 - 1 / numpy.sqrt(num_incoh))
1664 (1 - 1 / numpy.sqrt(num_incoh))
1618
1665
1619 # Remocion de Interferencia en el Cross Spectra
1666 # Remocion de Interferencia en el Cross Spectra
1620 if jcspectra is None:
1667 if jcspectra is None:
1621 return jspectra, jcspectra
1668 return jspectra, jcspectra
1622 num_pairs = int(jcspectra.size / (num_prof * num_hei))
1669 num_pairs = int(jcspectra.size / (num_prof * num_hei))
1623 jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
1670 jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
1624
1671
1625 for ip in range(num_pairs):
1672 for ip in range(num_pairs):
1626
1673
1627 #-------------------------------------------
1674 #-------------------------------------------
1628
1675
1629 cspower = numpy.abs(jcspectra[ip, mask_prof, :])
1676 cspower = numpy.abs(jcspectra[ip, mask_prof, :])
1630 cspower = cspower[:, hei_interf]
1677 cspower = cspower[:, hei_interf]
1631 cspower = cspower.sum(axis=0)
1678 cspower = cspower.sum(axis=0)
1632
1679
1633 cspsort = cspower.ravel().argsort()
1680 cspsort = cspower.ravel().argsort()
1634 junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
1681 junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
1635 offhei_interf, nhei_interf + offhei_interf))]]]
1682 offhei_interf, nhei_interf + offhei_interf))]]]
1636 junkcspc_interf = junkcspc_interf.transpose()
1683 junkcspc_interf = junkcspc_interf.transpose()
1637 jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
1684 jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
1638
1685
1639 ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
1686 ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
1640
1687
1641 median_real = int(numpy.median(numpy.real(
1688 median_real = int(numpy.median(numpy.real(
1642 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1689 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1643 median_imag = int(numpy.median(numpy.imag(
1690 median_imag = int(numpy.median(numpy.imag(
1644 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1691 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1645 comp_mask_prof = [int(e) for e in comp_mask_prof]
1692 comp_mask_prof = [int(e) for e in comp_mask_prof]
1646 junkcspc_interf[comp_mask_prof, :] = numpy.complex(
1693 junkcspc_interf[comp_mask_prof, :] = numpy.complex(
1647 median_real, median_imag)
1694 median_real, median_imag)
1648
1695
1649 for iprof in range(num_prof):
1696 for iprof in range(num_prof):
1650 ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
1697 ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
1651 jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf // 2]]
1698 jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf // 2]]
1652
1699
1653 # Removiendo la Interferencia
1700 # Removiendo la Interferencia
1654 jcspectra[ip, :, ind_hei] = jcspectra[ip,
1701 jcspectra[ip, :, ind_hei] = jcspectra[ip,
1655 :, ind_hei] - jcspc_interf
1702 :, ind_hei] - jcspc_interf
1656
1703
1657 ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
1704 ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
1658 maxid = ListAux.index(max(ListAux))
1705 maxid = ListAux.index(max(ListAux))
1659
1706
1660 ind = numpy.array([-2, -1, 1, 2])
1707 ind = numpy.array([-2, -1, 1, 2])
1661 xx = numpy.zeros([4, 4])
1708 xx = numpy.zeros([4, 4])
1662
1709
1663 for id1 in range(4):
1710 for id1 in range(4):
1664 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1711 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1665
1712
1666 xx_inv = numpy.linalg.inv(xx)
1713 xx_inv = numpy.linalg.inv(xx)
1667 xx = xx_inv[:, 0]
1714 xx = xx_inv[:, 0]
1668
1715
1669 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1716 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1670 yy = jcspectra[ip, mask_prof[ind], :]
1717 yy = jcspectra[ip, mask_prof[ind], :]
1671 jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
1718 jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
1672
1719
1673 # Guardar Resultados
1720 # Guardar Resultados
1674 self.dataOut.data_spc = jspectra
1721 self.dataOut.data_spc = jspectra
1675 self.dataOut.data_cspc = jcspectra
1722 self.dataOut.data_cspc = jcspectra
1676
1723
1677 return 1
1724 return 1
1678
1725
1679 def run(self, dataOut, interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None, mode=1):
1726 def run(self, dataOut, interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None, mode=1):
1680
1727
1681 self.dataOut = dataOut
1728 self.dataOut = dataOut
1682
1729
1683 if mode == 1:
1730 if mode == 1:
1684 self.removeInterference(interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None)
1731 self.removeInterference(interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None)
1685 elif mode == 2:
1732 elif mode == 2:
1686 self.removeInterference2()
1733 self.removeInterference2()
1687
1734
1688 return self.dataOut
1735 return self.dataOut
1689
1736
1690
1737
1691 class IncohInt(Operation):
1738 class IncohInt(Operation):
1692
1739
1693 __profIndex = 0
1740 __profIndex = 0
1694 __withOverapping = False
1741 __withOverapping = False
1695
1742
1696 __byTime = False
1743 __byTime = False
1697 __initime = None
1744 __initime = None
1698 __lastdatatime = None
1745 __lastdatatime = None
1699 __integrationtime = None
1746 __integrationtime = None
1700
1747
1701 __buffer_spc = None
1748 __buffer_spc = None
1702 __buffer_cspc = None
1749 __buffer_cspc = None
1703 __buffer_dc = None
1750 __buffer_dc = None
1704
1751
1705 __dataReady = False
1752 __dataReady = False
1706
1753
1707 __timeInterval = None
1754 __timeInterval = None
1708
1755 incohInt = 0
1756 nOutliers = 0
1709 n = None
1757 n = None
1710
1758
1711 def __init__(self):
1759 def __init__(self):
1712
1760
1713 Operation.__init__(self)
1761 Operation.__init__(self)
1714
1762
1715 def setup(self, n=None, timeInterval=None, overlapping=False):
1763 def setup(self, n=None, timeInterval=None, overlapping=False):
1716 """
1764 """
1717 Set the parameters of the integration class.
1765 Set the parameters of the integration class.
1718
1766
1719 Inputs:
1767 Inputs:
1720
1768
1721 n : Number of coherent integrations
1769 n : Number of coherent integrations
1722 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1770 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1723 overlapping :
1771 overlapping :
1724
1772
1725 """
1773 """
1726
1774
1727 self.__initime = None
1775 self.__initime = None
1728 self.__lastdatatime = 0
1776 self.__lastdatatime = 0
1729
1777
1730 self.__buffer_spc = 0
1778 self.__buffer_spc = 0
1731 self.__buffer_cspc = 0
1779 self.__buffer_cspc = 0
1732 self.__buffer_dc = 0
1780 self.__buffer_dc = 0
1733
1781
1734 self.__profIndex = 0
1782 self.__profIndex = 0
1735 self.__dataReady = False
1783 self.__dataReady = False
1736 self.__byTime = False
1784 self.__byTime = False
1737
1785 self.incohInt = 0
1786 self.nOutliers = 0
1738 if n is None and timeInterval is None:
1787 if n is None and timeInterval is None:
1739 raise ValueError("n or timeInterval should be specified ...")
1788 raise ValueError("n or timeInterval should be specified ...")
1740
1789
1741 if n is not None:
1790 if n is not None:
1742 self.n = int(n)
1791 self.n = int(n)
1743 else:
1792 else:
1744
1793
1745 self.__integrationtime = int(timeInterval)
1794 self.__integrationtime = int(timeInterval)
1746 self.n = None
1795 self.n = None
1747 self.__byTime = True
1796 self.__byTime = True
1748
1797
1749 def putData(self, data_spc, data_cspc, data_dc):
1798 def putData(self, data_spc, data_cspc, data_dc):
1750 """
1799 """
1751 Add a profile to the __buffer_spc and increase in one the __profileIndex
1800 Add a profile to the __buffer_spc and increase in one the __profileIndex
1752
1801
1753 """
1802 """
1754 if data_spc.all() == numpy.nan :
1803 if data_spc.all() == numpy.nan :
1755 print("nan ")
1804 print("nan ")
1756 return
1805 return
1757 self.__buffer_spc += data_spc
1806 self.__buffer_spc += data_spc
1758
1807
1759 if data_cspc is None:
1808 if data_cspc is None:
1760 self.__buffer_cspc = None
1809 self.__buffer_cspc = None
1761 else:
1810 else:
1762 self.__buffer_cspc += data_cspc
1811 self.__buffer_cspc += data_cspc
1763
1812
1764 if data_dc is None:
1813 if data_dc is None:
1765 self.__buffer_dc = None
1814 self.__buffer_dc = None
1766 else:
1815 else:
1767 self.__buffer_dc += data_dc
1816 self.__buffer_dc += data_dc
1768
1817
1769 self.__profIndex += 1
1818 self.__profIndex += 1
1770
1819
1771 return
1820 return
1772
1821
1773 def pushData(self):
1822 def pushData(self):
1774 """
1823 """
1775 Return the sum of the last profiles and the profiles used in the sum.
1824 Return the sum of the last profiles and the profiles used in the sum.
1776
1825
1777 Affected:
1826 Affected:
1778
1827
1779 self.__profileIndex
1828 self.__profileIndex
1780
1829
1781 """
1830 """
1782
1831
1783 data_spc = self.__buffer_spc
1832 data_spc = self.__buffer_spc
1784 data_cspc = self.__buffer_cspc
1833 data_cspc = self.__buffer_cspc
1785 data_dc = self.__buffer_dc
1834 data_dc = self.__buffer_dc
1786 n = self.__profIndex
1835 n = self.__profIndex
1787
1836
1788 self.__buffer_spc = 0
1837 self.__buffer_spc = 0
1789 self.__buffer_cspc = 0
1838 self.__buffer_cspc = 0
1790 self.__buffer_dc = 0
1839 self.__buffer_dc = 0
1791 self.__profIndex = 0
1840
1792
1841
1793 return data_spc, data_cspc, data_dc, n
1842 return data_spc, data_cspc, data_dc, n
1794
1843
1795 def byProfiles(self, *args):
1844 def byProfiles(self, *args):
1796
1845
1797 self.__dataReady = False
1846 self.__dataReady = False
1798 avgdata_spc = None
1847 avgdata_spc = None
1799 avgdata_cspc = None
1848 avgdata_cspc = None
1800 avgdata_dc = None
1849 avgdata_dc = None
1801
1850
1802 self.putData(*args)
1851 self.putData(*args)
1803
1852
1804 if self.__profIndex == self.n:
1853 if self.__profIndex == self.n:
1805
1854
1806 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1855 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1807 self.n = n
1856 self.n = n
1808 self.__dataReady = True
1857 self.__dataReady = True
1809
1858
1810 return avgdata_spc, avgdata_cspc, avgdata_dc
1859 return avgdata_spc, avgdata_cspc, avgdata_dc
1811
1860
1812 def byTime(self, datatime, *args):
1861 def byTime(self, datatime, *args):
1813
1862
1814 self.__dataReady = False
1863 self.__dataReady = False
1815 avgdata_spc = None
1864 avgdata_spc = None
1816 avgdata_cspc = None
1865 avgdata_cspc = None
1817 avgdata_dc = None
1866 avgdata_dc = None
1818
1867
1819 self.putData(*args)
1868 self.putData(*args)
1820
1869
1821 if (datatime - self.__initime) >= self.__integrationtime:
1870 if (datatime - self.__initime) >= self.__integrationtime:
1822 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1871 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1823 self.n = n
1872 self.n = n
1824 self.__dataReady = True
1873 self.__dataReady = True
1825
1874
1826 return avgdata_spc, avgdata_cspc, avgdata_dc
1875 return avgdata_spc, avgdata_cspc, avgdata_dc
1827
1876
1828 def integrate(self, datatime, *args):
1877 def integrate(self, datatime, *args):
1829
1878
1830 if self.__profIndex == 0:
1879 if self.__profIndex == 0:
1831 self.__initime = datatime
1880 self.__initime = datatime
1832
1881
1833 if self.__byTime:
1882 if self.__byTime:
1834 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
1883 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
1835 datatime, *args)
1884 datatime, *args)
1836 else:
1885 else:
1837 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1886 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1838
1887
1839 if not self.__dataReady:
1888 if not self.__dataReady:
1840 return None, None, None, None
1889 return None, None, None, None
1841
1890
1842 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
1891 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
1843
1892
1844 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
1893 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
1845 if n == 1:
1894 if n == 1:
1846 return dataOut
1895 return dataOut
1847
1896
1897 if dataOut.flagNoData == True:
1898 return dataOut
1899
1848 dataOut.flagNoData = True
1900 dataOut.flagNoData = True
1849
1901
1850 if not self.isConfig:
1902 if not self.isConfig:
1851 self.setup(n, timeInterval, overlapping)
1903 self.setup(n, timeInterval, overlapping)
1852 self.isConfig = True
1904 self.isConfig = True
1853
1905
1854 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
1906 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
1855 dataOut.data_spc,
1907 dataOut.data_spc,
1856 dataOut.data_cspc,
1908 dataOut.data_cspc,
1857 dataOut.data_dc)
1909 dataOut.data_dc)
1858
1910 self.incohInt += dataOut.nIncohInt
1911 self.nOutliers += dataOut.data_outlier
1859 if self.__dataReady:
1912 if self.__dataReady:
1860
1913
1861 dataOut.data_spc = avgdata_spc
1914 dataOut.data_spc = avgdata_spc
1862 dataOut.data_cspc = avgdata_cspc
1915 dataOut.data_cspc = avgdata_cspc
1863 dataOut.data_dc = avgdata_dc
1916 dataOut.data_dc = avgdata_dc
1864 dataOut.nIncohInt *= self.n
1917 dataOut.nIncohInt = self.incohInt
1918 dataOut.data_outlier = self.nOutliers
1865 dataOut.utctime = avgdatatime
1919 dataOut.utctime = avgdatatime
1866 dataOut.flagNoData = False
1920 dataOut.flagNoData = False
1921 dataOut.max_nIncohInt = self.__profIndex
1922 self.incohInt = 0
1923 self.nOutliers = 0
1924 self.__profIndex = 0
1867
1925
1868 return dataOut
1926 return dataOut
1869
1927
1870 class dopplerFlip(Operation):
1928 class dopplerFlip(Operation):
1871
1929
1872 def run(self, dataOut):
1930 def run(self, dataOut):
1873 # arreglo 1: (num_chan, num_profiles, num_heights)
1931 # arreglo 1: (num_chan, num_profiles, num_heights)
1874 self.dataOut = dataOut
1932 self.dataOut = dataOut
1875 # JULIA-oblicua, indice 2
1933 # JULIA-oblicua, indice 2
1876 # arreglo 2: (num_profiles, num_heights)
1934 # arreglo 2: (num_profiles, num_heights)
1877 jspectra = self.dataOut.data_spc[2]
1935 jspectra = self.dataOut.data_spc[2]
1878 jspectra_tmp = numpy.zeros(jspectra.shape)
1936 jspectra_tmp = numpy.zeros(jspectra.shape)
1879 num_profiles = jspectra.shape[0]
1937 num_profiles = jspectra.shape[0]
1880 freq_dc = int(num_profiles / 2)
1938 freq_dc = int(num_profiles / 2)
1881 # Flip con for
1939 # Flip con for
1882 for j in range(num_profiles):
1940 for j in range(num_profiles):
1883 jspectra_tmp[num_profiles-j-1]= jspectra[j]
1941 jspectra_tmp[num_profiles-j-1]= jspectra[j]
1884 # Intercambio perfil de DC con perfil inmediato anterior
1942 # Intercambio perfil de DC con perfil inmediato anterior
1885 jspectra_tmp[freq_dc-1]= jspectra[freq_dc-1]
1943 jspectra_tmp[freq_dc-1]= jspectra[freq_dc-1]
1886 jspectra_tmp[freq_dc]= jspectra[freq_dc]
1944 jspectra_tmp[freq_dc]= jspectra[freq_dc]
1887 # canal modificado es re-escrito en el arreglo de canales
1945 # canal modificado es re-escrito en el arreglo de canales
1888 self.dataOut.data_spc[2] = jspectra_tmp
1946 self.dataOut.data_spc[2] = jspectra_tmp
1889
1947
1890 return self.dataOut
1948 return self.dataOut
Show file before | Show file after | Hide comments
@@ -1,2234 +1,2361
1 import sys
1 import sys
2 import numpy,math
2 import numpy,math
3 from scipy import interpolate
3 from scipy import interpolate
4 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
4 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
5 from schainpy.model.data.jrodata import Voltage,hildebrand_sekhon
5 from schainpy.model.data.jrodata import Voltage,hildebrand_sekhon
6 from schainpy.utils import log
6 from schainpy.utils import log
7 from schainpy.model.io.utils import getHei_index
7 from schainpy.model.io.utils import getHei_index
8 from time import time
8 from time import time
9 import datetime
9 import datetime
10 import numpy
10 import numpy
11 import copy
11 #import copy
12 from schainpy.model.data import _noise
12
13
13 class VoltageProc(ProcessingUnit):
14 class VoltageProc(ProcessingUnit):
14
15
15 def __init__(self):
16 def __init__(self):
16
17
17 ProcessingUnit.__init__(self)
18 ProcessingUnit.__init__(self)
18
19
19 self.dataOut = Voltage()
20 self.dataOut = Voltage()
20 self.flip = 1
21 self.flip = 1
21 self.setupReq = False
22 self.setupReq = False
22
23
23 def run(self):
24 def run(self):
24 #print("running volt proc")
25 #print("running volt proc")
25 if self.dataIn.type == 'AMISR':
26 if self.dataIn.type == 'AMISR':
26 self.__updateObjFromAmisrInput()
27 self.__updateObjFromAmisrInput()
27
28
28 if self.dataOut.buffer_empty:
29 if self.dataOut.buffer_empty:
29 if self.dataIn.type == 'Voltage':
30 if self.dataIn.type == 'Voltage':
30 self.dataOut.copy(self.dataIn)
31 self.dataOut.copy(self.dataIn)
31 #print("new volts reading")
32 #print("new volts reading")
32
33
33
34
34 def __updateObjFromAmisrInput(self):
35 def __updateObjFromAmisrInput(self):
35
36
36 self.dataOut.timeZone = self.dataIn.timeZone
37 self.dataOut.timeZone = self.dataIn.timeZone
37 self.dataOut.dstFlag = self.dataIn.dstFlag
38 self.dataOut.dstFlag = self.dataIn.dstFlag
38 self.dataOut.errorCount = self.dataIn.errorCount
39 self.dataOut.errorCount = self.dataIn.errorCount
39 self.dataOut.useLocalTime = self.dataIn.useLocalTime
40 self.dataOut.useLocalTime = self.dataIn.useLocalTime
40
41
41 self.dataOut.flagNoData = self.dataIn.flagNoData
42 self.dataOut.flagNoData = self.dataIn.flagNoData
42 self.dataOut.data = self.dataIn.data
43 self.dataOut.data = self.dataIn.data
43 self.dataOut.utctime = self.dataIn.utctime
44 self.dataOut.utctime = self.dataIn.utctime
44 self.dataOut.channelList = self.dataIn.channelList
45 self.dataOut.channelList = self.dataIn.channelList
45 #self.dataOut.timeInterval = self.dataIn.timeInterval
46 #self.dataOut.timeInterval = self.dataIn.timeInterval
46 self.dataOut.heightList = self.dataIn.heightList
47 self.dataOut.heightList = self.dataIn.heightList
47 self.dataOut.nProfiles = self.dataIn.nProfiles
48 self.dataOut.nProfiles = self.dataIn.nProfiles
48
49
49 self.dataOut.nCohInt = self.dataIn.nCohInt
50 self.dataOut.nCohInt = self.dataIn.nCohInt
50 self.dataOut.ippSeconds = self.dataIn.ippSeconds
51 self.dataOut.ippSeconds = self.dataIn.ippSeconds
51 self.dataOut.frequency = self.dataIn.frequency
52 self.dataOut.frequency = self.dataIn.frequency
52
53
53 self.dataOut.azimuth = self.dataIn.azimuth
54 self.dataOut.azimuth = self.dataIn.azimuth
54 self.dataOut.zenith = self.dataIn.zenith
55 self.dataOut.zenith = self.dataIn.zenith
55
56
56 self.dataOut.beam.codeList = self.dataIn.beam.codeList
57 self.dataOut.beam.codeList = self.dataIn.beam.codeList
57 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
58 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
58 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
59 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
59
60
60
61
61 class selectChannels(Operation):
62 class selectChannels(Operation):
62
63
63 def run(self, dataOut, channelList=None):
64 def run(self, dataOut, channelList=None):
64 self.channelList = channelList
65 self.channelList = channelList
65 if self.channelList == None:
66 if self.channelList == None:
66 print("Missing channelList")
67 print("Missing channelList")
67 return dataOut
68 return dataOut
68 channelIndexList = []
69 channelIndexList = []
69
70
70 if type(dataOut.channelList) is not list: #leer array desde HDF5
71 if type(dataOut.channelList) is not list: #leer array desde HDF5
71 try:
72 try:
72 dataOut.channelList = dataOut.channelList.tolist()
73 dataOut.channelList = dataOut.channelList.tolist()
73 except Exception as e:
74 except Exception as e:
74 print("Select Channels: ",e)
75 print("Select Channels: ",e)
75 for channel in self.channelList:
76 for channel in self.channelList:
76 if channel not in dataOut.channelList:
77 if channel not in dataOut.channelList:
77 raise ValueError("Channel %d is not in %s" %(channel, str(dataOut.channelList)))
78 raise ValueError("Channel %d is not in %s" %(channel, str(dataOut.channelList)))
78
79
79 index = dataOut.channelList.index(channel)
80 index = dataOut.channelList.index(channel)
80 channelIndexList.append(index)
81 channelIndexList.append(index)
81 dataOut = self.selectChannelsByIndex(dataOut,channelIndexList)
82 dataOut = self.selectChannelsByIndex(dataOut,channelIndexList)
82 return dataOut
83 return dataOut
83
84
84 def selectChannelsByIndex(self, dataOut, channelIndexList):
85 def selectChannelsByIndex(self, dataOut, channelIndexList):
85 """
86 """
86 Selecciona un bloque de datos en base a canales segun el channelIndexList
87 Selecciona un bloque de datos en base a canales segun el channelIndexList
87
88
88 Input:
89 Input:
89 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
90 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
90
91
91 Affected:
92 Affected:
92 dataOut.data
93 dataOut.data
93 dataOut.channelIndexList
94 dataOut.channelIndexList
94 dataOut.nChannels
95 dataOut.nChannels
95 dataOut.m_ProcessingHeader.totalSpectra
96 dataOut.m_ProcessingHeader.totalSpectra
96 dataOut.systemHeaderObj.numChannels
97 dataOut.systemHeaderObj.numChannels
97 dataOut.m_ProcessingHeader.blockSize
98 dataOut.m_ProcessingHeader.blockSize
98
99
99 Return:
100 Return:
100 None
101 None
101 """
102 """
102 #print("selectChannelsByIndex")
103 #print("selectChannelsByIndex")
103 # for channelIndex in channelIndexList:
104 # for channelIndex in channelIndexList:
104 # if channelIndex not in dataOut.channelIndexList:
105 # if channelIndex not in dataOut.channelIndexList:
105 # raise ValueError("The value %d in channelIndexList is not valid" %channelIndex)
106 # raise ValueError("The value %d in channelIndexList is not valid" %channelIndex)
106
107
107 if dataOut.type == 'Voltage':
108 if dataOut.type == 'Voltage':
108 if dataOut.flagDataAsBlock:
109 if dataOut.flagDataAsBlock:
109 """
110 """
110 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
111 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
111 """
112 """
112 data = dataOut.data[channelIndexList,:,:]
113 data = dataOut.data[channelIndexList,:,:]
113 else:
114 else:
114 data = dataOut.data[channelIndexList,:]
115 data = dataOut.data[channelIndexList,:]
115
116
116 dataOut.data = data
117 dataOut.data = data
117 # dataOut.channelList = [dataOut.channelList[i] for i in channelIndexList]
118 # dataOut.channelList = [dataOut.channelList[i] for i in channelIndexList]
118 dataOut.channelList = range(len(channelIndexList))
119 dataOut.channelList = range(len(channelIndexList))
119
120
120 elif dataOut.type == 'Spectra':
121 elif dataOut.type == 'Spectra':
121 if hasattr(dataOut, 'data_spc'):
122 if hasattr(dataOut, 'data_spc'):
122 if dataOut.data_spc is None:
123 if dataOut.data_spc is None:
123 raise ValueError("data_spc is None")
124 raise ValueError("data_spc is None")
124 return dataOut
125 return dataOut
125 else:
126 else:
126 data_spc = dataOut.data_spc[channelIndexList, :]
127 data_spc = dataOut.data_spc[channelIndexList, :]
127 dataOut.data_spc = data_spc
128 dataOut.data_spc = data_spc
128
129
129 # if hasattr(dataOut, 'data_dc') :# and
130 # if hasattr(dataOut, 'data_dc') :# and
130 # if dataOut.data_dc is None:
131 # if dataOut.data_dc is None:
131 # raise ValueError("data_dc is None")
132 # raise ValueError("data_dc is None")
132 # return dataOut
133 # return dataOut
133 # else:
134 # else:
134 # data_dc = dataOut.data_dc[channelIndexList, :]
135 # data_dc = dataOut.data_dc[channelIndexList, :]
135 # dataOut.data_dc = data_dc
136 # dataOut.data_dc = data_dc
136 # dataOut.channelList = [dataOut.channelList[i] for i in channelIndexList]
137 # dataOut.channelList = [dataOut.channelList[i] for i in channelIndexList]
137 dataOut.channelList = channelIndexList
138 dataOut.channelList = channelIndexList
138 dataOut = self.__selectPairsByChannel(dataOut,channelIndexList)
139 dataOut = self.__selectPairsByChannel(dataOut,channelIndexList)
139
140
140 return dataOut
141 return dataOut
141
142
142 def __selectPairsByChannel(self, dataOut, channelList=None):
143 def __selectPairsByChannel(self, dataOut, channelList=None):
143 #print("__selectPairsByChannel")
144 #print("__selectPairsByChannel")
144 if channelList == None:
145 if channelList == None:
145 return
146 return
146
147
147 pairsIndexListSelected = []
148 pairsIndexListSelected = []
148 for pairIndex in dataOut.pairsIndexList:
149 for pairIndex in dataOut.pairsIndexList:
149 # First pair
150 # First pair
150 if dataOut.pairsList[pairIndex][0] not in channelList:
151 if dataOut.pairsList[pairIndex][0] not in channelList:
151 continue
152 continue
152 # Second pair
153 # Second pair
153 if dataOut.pairsList[pairIndex][1] not in channelList:
154 if dataOut.pairsList[pairIndex][1] not in channelList:
154 continue
155 continue
155
156
156 pairsIndexListSelected.append(pairIndex)
157 pairsIndexListSelected.append(pairIndex)
157 if not pairsIndexListSelected:
158 if not pairsIndexListSelected:
158 dataOut.data_cspc = None
159 dataOut.data_cspc = None
159 dataOut.pairsList = []
160 dataOut.pairsList = []
160 return
161 return
161
162
162 dataOut.data_cspc = dataOut.data_cspc[pairsIndexListSelected]
163 dataOut.data_cspc = dataOut.data_cspc[pairsIndexListSelected]
163 dataOut.pairsList = [dataOut.pairsList[i]
164 dataOut.pairsList = [dataOut.pairsList[i]
164 for i in pairsIndexListSelected]
165 for i in pairsIndexListSelected]
165
166
166 return dataOut
167 return dataOut
167
168
168 class selectHeights(Operation):
169 class selectHeights(Operation):
169
170
170 def run(self, dataOut, minHei=None, maxHei=None, minIndex=None, maxIndex=None):
171 def run(self, dataOut, minHei=None, maxHei=None, minIndex=None, maxIndex=None):
171 """
172 """
172 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
173 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
173 minHei <= height <= maxHei
174 minHei <= height <= maxHei
174
175
175 Input:
176 Input:
176 minHei : valor minimo de altura a considerar
177 minHei : valor minimo de altura a considerar
177 maxHei : valor maximo de altura a considerar
178 maxHei : valor maximo de altura a considerar
178
179
179 Affected:
180 Affected:
180 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
181 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
181
182
182 Return:
183 Return:
183 1 si el metodo se ejecuto con exito caso contrario devuelve 0
184 1 si el metodo se ejecuto con exito caso contrario devuelve 0
184 """
185 """
185
186
186 self.dataOut = dataOut
187 self.dataOut = dataOut
187
188
188 if minHei and maxHei:
189 if minHei and maxHei:
189
190
190 if (minHei < dataOut.heightList[0]):
191 if (minHei < dataOut.heightList[0]):
191 minHei = dataOut.heightList[0]
192 minHei = dataOut.heightList[0]
192
193
193 if (maxHei > dataOut.heightList[-1]):
194 if (maxHei > dataOut.heightList[-1]):
194 maxHei = dataOut.heightList[-1]
195 maxHei = dataOut.heightList[-1]
195
196
196 minIndex = 0
197 minIndex = 0
197 maxIndex = 0
198 maxIndex = 0
198 heights = dataOut.heightList
199 heights = dataOut.heightList
199
200
200 inda = numpy.where(heights >= minHei)
201 inda = numpy.where(heights >= minHei)
201 indb = numpy.where(heights <= maxHei)
202 indb = numpy.where(heights <= maxHei)
202
203
203 try:
204 try:
204 minIndex = inda[0][0]
205 minIndex = inda[0][0]
205 except:
206 except:
206 minIndex = 0
207 minIndex = 0
207
208
208 try:
209 try:
209 maxIndex = indb[0][-1]
210 maxIndex = indb[0][-1]
210 except:
211 except:
211 maxIndex = len(heights)
212 maxIndex = len(heights)
212
213
213 self.selectHeightsByIndex(minIndex, maxIndex)
214 self.selectHeightsByIndex(minIndex, maxIndex)
214
215
215 return dataOut
216 return dataOut
216
217
217 def selectHeightsByIndex(self, minIndex, maxIndex):
218 def selectHeightsByIndex(self, minIndex, maxIndex):
218 """
219 """
219 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
220 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
220 minIndex <= index <= maxIndex
221 minIndex <= index <= maxIndex
221
222
222 Input:
223 Input:
223 minIndex : valor de indice minimo de altura a considerar
224 minIndex : valor de indice minimo de altura a considerar
224 maxIndex : valor de indice maximo de altura a considerar
225 maxIndex : valor de indice maximo de altura a considerar
225
226
226 Affected:
227 Affected:
227 self.dataOut.data
228 self.dataOut.data
228 self.dataOut.heightList
229 self.dataOut.heightList
229
230
230 Return:
231 Return:
231 1 si el metodo se ejecuto con exito caso contrario devuelve 0
232 1 si el metodo se ejecuto con exito caso contrario devuelve 0
232 """
233 """
233
234
234 if self.dataOut.type == 'Voltage':
235 if self.dataOut.type == 'Voltage':
235 if (minIndex < 0) or (minIndex > maxIndex):
236 if (minIndex < 0) or (minIndex > maxIndex):
236 raise ValueError("Height index range (%d,%d) is not valid" % (minIndex, maxIndex))
237 raise ValueError("Height index range (%d,%d) is not valid" % (minIndex, maxIndex))
237
238
238 if (maxIndex >= self.dataOut.nHeights):
239 if (maxIndex >= self.dataOut.nHeights):
239 maxIndex = self.dataOut.nHeights
240 maxIndex = self.dataOut.nHeights
240
241
241 #voltage
242 #voltage
242 if self.dataOut.flagDataAsBlock:
243 if self.dataOut.flagDataAsBlock:
243 """
244 """
244 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
245 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
245 """
246 """
246 data = self.dataOut.data[:,:, minIndex:maxIndex]
247 data = self.dataOut.data[:,:, minIndex:maxIndex]
247 else:
248 else:
248 data = self.dataOut.data[:, minIndex:maxIndex]
249 data = self.dataOut.data[:, minIndex:maxIndex]
249
250
250 # firstHeight = self.dataOut.heightList[minIndex]
251 # firstHeight = self.dataOut.heightList[minIndex]
251
252
252 self.dataOut.data = data
253 self.dataOut.data = data
253 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
254 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
254
255
255 if self.dataOut.nHeights <= 1:
256 if self.dataOut.nHeights <= 1:
256 raise ValueError("selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights))
257 raise ValueError("selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights))
257 elif self.dataOut.type == 'Spectra':
258 elif self.dataOut.type == 'Spectra':
258 if (minIndex < 0) or (minIndex > maxIndex):
259 if (minIndex < 0) or (minIndex > maxIndex):
259 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (
260 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (
260 minIndex, maxIndex))
261 minIndex, maxIndex))
261
262
262 if (maxIndex >= self.dataOut.nHeights):
263 if (maxIndex >= self.dataOut.nHeights):
263 maxIndex = self.dataOut.nHeights - 1
264 maxIndex = self.dataOut.nHeights - 1
264
265
265 # Spectra
266 # Spectra
266 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
267 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
267
268
268 data_cspc = None
269 data_cspc = None
269 if self.dataOut.data_cspc is not None:
270 if self.dataOut.data_cspc is not None:
270 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
271 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
271
272
272 data_dc = None
273 data_dc = None
273 if self.dataOut.data_dc is not None:
274 if self.dataOut.data_dc is not None:
274 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
275 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
275
276
276 self.dataOut.data_spc = data_spc
277 self.dataOut.data_spc = data_spc
277 self.dataOut.data_cspc = data_cspc
278 self.dataOut.data_cspc = data_cspc
278 self.dataOut.data_dc = data_dc
279 self.dataOut.data_dc = data_dc
279
280
280 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
281 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
281
282
282 return 1
283 return 1
283
284
284
285
285 class filterByHeights(Operation):
286 class filterByHeights(Operation):
286
287
287 def run(self, dataOut, window):
288 def run(self, dataOut, window):
288
289
289 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
290 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
290
291
291 if window == None:
292 if window == None:
292 window = (dataOut.radarControllerHeaderObj.txA/dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
293 window = (dataOut.radarControllerHeaderObj.txA/dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
293
294
294 newdelta = deltaHeight * window
295 newdelta = deltaHeight * window
295 r = dataOut.nHeights % window
296 r = dataOut.nHeights % window
296 newheights = (dataOut.nHeights-r)/window
297 newheights = (dataOut.nHeights-r)/window
297
298
298 if newheights <= 1:
299 if newheights <= 1:
299 raise ValueError("filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(dataOut.nHeights, window))
300 raise ValueError("filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(dataOut.nHeights, window))
300
301
301 if dataOut.flagDataAsBlock:
302 if dataOut.flagDataAsBlock:
302 """
303 """
303 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
304 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
304 """
305 """
305 buffer = dataOut.data[:, :, 0:int(dataOut.nHeights-r)]
306 buffer = dataOut.data[:, :, 0:int(dataOut.nHeights-r)]
306 buffer = buffer.reshape(dataOut.nChannels, dataOut.nProfiles, int(dataOut.nHeights/window), window)
307 buffer = buffer.reshape(dataOut.nChannels, dataOut.nProfiles, int(dataOut.nHeights/window), window)
307 buffer = numpy.sum(buffer,3)
308 buffer = numpy.sum(buffer,3)
308
309
309 else:
310 else:
310 buffer = dataOut.data[:,0:int(dataOut.nHeights-r)]
311 buffer = dataOut.data[:,0:int(dataOut.nHeights-r)]
311 buffer = buffer.reshape(dataOut.nChannels,int(dataOut.nHeights/window),int(window))
312 buffer = buffer.reshape(dataOut.nChannels,int(dataOut.nHeights/window),int(window))
312 buffer = numpy.sum(buffer,2)
313 buffer = numpy.sum(buffer,2)
313
314
314 dataOut.data = buffer
315 dataOut.data = buffer
315 dataOut.heightList = dataOut.heightList[0] + numpy.arange( newheights )*newdelta
316 dataOut.heightList = dataOut.heightList[0] + numpy.arange( newheights )*newdelta
316 dataOut.windowOfFilter = window
317 dataOut.windowOfFilter = window
317
318
318 return dataOut
319 return dataOut
319
320
320
321
321 class setH0(Operation):
322 class setH0(Operation):
322
323
323 def run(self, dataOut, h0, deltaHeight = None):
324 def run(self, dataOut, h0, deltaHeight = None):
324
325
325 if not deltaHeight:
326 if not deltaHeight:
326 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
327 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
327
328
328 nHeights = dataOut.nHeights
329 nHeights = dataOut.nHeights
329
330
330 newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
331 newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
331
332
332 dataOut.heightList = newHeiRange
333 dataOut.heightList = newHeiRange
333
334
334 return dataOut
335 return dataOut
335
336
336
337
337 class deFlip(Operation):
338 class deFlip(Operation):
338
339
339 def run(self, dataOut, channelList = []):
340 def run(self, dataOut, channelList = []):
340
341
341 data = dataOut.data.copy()
342 data = dataOut.data.copy()
342
343
343 if dataOut.flagDataAsBlock:
344 if dataOut.flagDataAsBlock:
344 flip = self.flip
345 flip = self.flip
345 profileList = list(range(dataOut.nProfiles))
346 profileList = list(range(dataOut.nProfiles))
346
347
347 if not channelList:
348 if not channelList:
348 for thisProfile in profileList:
349 for thisProfile in profileList:
349 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
350 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
350 flip *= -1.0
351 flip *= -1.0
351 else:
352 else:
352 for thisChannel in channelList:
353 for thisChannel in channelList:
353 if thisChannel not in dataOut.channelList:
354 if thisChannel not in dataOut.channelList:
354 continue
355 continue
355
356
356 for thisProfile in profileList:
357 for thisProfile in profileList:
357 data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
358 data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
358 flip *= -1.0
359 flip *= -1.0
359
360
360 self.flip = flip
361 self.flip = flip
361
362
362 else:
363 else:
363 if not channelList:
364 if not channelList:
364 data[:,:] = data[:,:]*self.flip
365 data[:,:] = data[:,:]*self.flip
365 else:
366 else:
366 for thisChannel in channelList:
367 for thisChannel in channelList:
367 if thisChannel not in dataOut.channelList:
368 if thisChannel not in dataOut.channelList:
368 continue
369 continue
369
370
370 data[thisChannel,:] = data[thisChannel,:]*self.flip
371 data[thisChannel,:] = data[thisChannel,:]*self.flip
371
372
372 self.flip *= -1.
373 self.flip *= -1.
373
374
374 dataOut.data = data
375 dataOut.data = data
375
376
376 return dataOut
377 return dataOut
377
378
378
379
379 class setAttribute(Operation):
380 class setAttribute(Operation):
380 '''
381 '''
381 Set an arbitrary attribute(s) to dataOut
382 Set an arbitrary attribute(s) to dataOut
382 '''
383 '''
383
384
384 def __init__(self):
385 def __init__(self):
385
386
386 Operation.__init__(self)
387 Operation.__init__(self)
387 self._ready = False
388 self._ready = False
388
389
389 def run(self, dataOut, **kwargs):
390 def run(self, dataOut, **kwargs):
390
391
391 for key, value in kwargs.items():
392 for key, value in kwargs.items():
392 setattr(dataOut, key, value)
393 setattr(dataOut, key, value)
393
394
394 return dataOut
395 return dataOut
395
396
396
397
397 @MPDecorator
398 @MPDecorator
398 class printAttribute(Operation):
399 class printAttribute(Operation):
399 '''
400 '''
400 Print an arbitrary attribute of dataOut
401 Print an arbitrary attribute of dataOut
401 '''
402 '''
402
403
403 def __init__(self):
404 def __init__(self):
404
405
405 Operation.__init__(self)
406 Operation.__init__(self)
406
407
407 def run(self, dataOut, attributes):
408 def run(self, dataOut, attributes):
408
409
409 if isinstance(attributes, str):
410 if isinstance(attributes, str):
410 attributes = [attributes]
411 attributes = [attributes]
411 for attr in attributes:
412 for attr in attributes:
412 if hasattr(dataOut, attr):
413 if hasattr(dataOut, attr):
413 log.log(getattr(dataOut, attr), attr)
414 log.log(getattr(dataOut, attr), attr)
414
415
415
416
416 class interpolateHeights(Operation):
417 class interpolateHeights(Operation):
417
418
418 def run(self, dataOut, topLim, botLim):
419 def run(self, dataOut, topLim, botLim):
419 #69 al 72 para julia
420 #69 al 72 para julia
420 #82-84 para meteoros
421 #82-84 para meteoros
421 if len(numpy.shape(dataOut.data))==2:
422 if len(numpy.shape(dataOut.data))==2:
422 sampInterp = (dataOut.data[:,botLim-1] + dataOut.data[:,topLim+1])/2
423 sampInterp = (dataOut.data[:,botLim-1] + dataOut.data[:,topLim+1])/2
423 sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
424 sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
424 #dataOut.data[:,botLim:limSup+1] = sampInterp
425 #dataOut.data[:,botLim:limSup+1] = sampInterp
425 dataOut.data[:,botLim:topLim+1] = sampInterp
426 dataOut.data[:,botLim:topLim+1] = sampInterp
426 else:
427 else:
427 nHeights = dataOut.data.shape[2]
428 nHeights = dataOut.data.shape[2]
428 x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
429 x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
429 y = dataOut.data[:,:,list(range(botLim))+list(range(topLim+1,nHeights))]
430 y = dataOut.data[:,:,list(range(botLim))+list(range(topLim+1,nHeights))]
430 f = interpolate.interp1d(x, y, axis = 2)
431 f = interpolate.interp1d(x, y, axis = 2)
431 xnew = numpy.arange(botLim,topLim+1)
432 xnew = numpy.arange(botLim,topLim+1)
432 ynew = f(xnew)
433 ynew = f(xnew)
433 dataOut.data[:,:,botLim:topLim+1] = ynew
434 dataOut.data[:,:,botLim:topLim+1] = ynew
434
435
435 return dataOut
436 return dataOut
436
437
437
438
438 class CohInt(Operation):
439 class CohInt(Operation):
439
440
440 isConfig = False
441 isConfig = False
441 __profIndex = 0
442 __profIndex = 0
442 __byTime = False
443 __byTime = False
443 __initime = None
444 __initime = None
444 __lastdatatime = None
445 __lastdatatime = None
445 __integrationtime = None
446 __integrationtime = None
446 __buffer = None
447 __buffer = None
447 __bufferStride = []
448 __bufferStride = []
448 __dataReady = False
449 __dataReady = False
449 __profIndexStride = 0
450 __profIndexStride = 0
450 __dataToPutStride = False
451 __dataToPutStride = False
451 n = None
452 n = None
452
453
453 def __init__(self, **kwargs):
454 def __init__(self, **kwargs):
454
455
455 Operation.__init__(self, **kwargs)
456 Operation.__init__(self, **kwargs)
456
457
457 def setup(self, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False):
458 def setup(self, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False):
458 """
459 """
459 Set the parameters of the integration class.
460 Set the parameters of the integration class.
460
461
461 Inputs:
462 Inputs:
462
463
463 n : Number of coherent integrations
464 n : Number of coherent integrations
464 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
465 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
465 overlapping :
466 overlapping :
466 """
467 """
467
468
468 self.__initime = None
469 self.__initime = None
469 self.__lastdatatime = 0
470 self.__lastdatatime = 0
470 self.__buffer = None
471 self.__buffer = None
471 self.__dataReady = False
472 self.__dataReady = False
472 self.byblock = byblock
473 self.byblock = byblock
473 self.stride = stride
474 self.stride = stride
474
475
475 if n == None and timeInterval == None:
476 if n == None and timeInterval == None:
476 raise ValueError("n or timeInterval should be specified ...")
477 raise ValueError("n or timeInterval should be specified ...")
477
478
478 if n != None:
479 if n != None:
479 self.n = n
480 self.n = n
480 self.__byTime = False
481 self.__byTime = False
481 else:
482 else:
482 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
483 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
483 self.n = 9999
484 self.n = 9999
484 self.__byTime = True
485 self.__byTime = True
485
486
486 if overlapping:
487 if overlapping:
487 self.__withOverlapping = True
488 self.__withOverlapping = True
488 self.__buffer = None
489 self.__buffer = None
489 else:
490 else:
490 self.__withOverlapping = False
491 self.__withOverlapping = False
491 self.__buffer = 0
492 self.__buffer = 0
492
493
493 self.__profIndex = 0
494 self.__profIndex = 0
494
495
495 def putData(self, data):
496 def putData(self, data):
496
497
497 """
498 """
498 Add a profile to the __buffer and increase in one the __profileIndex
499 Add a profile to the __buffer and increase in one the __profileIndex
499
500
500 """
501 """
501
502
502 if not self.__withOverlapping:
503 if not self.__withOverlapping:
503 self.__buffer += data.copy()
504 self.__buffer += data.copy()
504 self.__profIndex += 1
505 self.__profIndex += 1
505 return
506 return
506
507
507 #Overlapping data
508 #Overlapping data
508 nChannels, nHeis = data.shape
509 nChannels, nHeis = data.shape
509 data = numpy.reshape(data, (1, nChannels, nHeis))
510 data = numpy.reshape(data, (1, nChannels, nHeis))
510
511
511 #If the buffer is empty then it takes the data value
512 #If the buffer is empty then it takes the data value
512 if self.__buffer is None:
513 if self.__buffer is None:
513 self.__buffer = data
514 self.__buffer = data
514 self.__profIndex += 1
515 self.__profIndex += 1
515 return
516 return
516
517
517 #If the buffer length is lower than n then stakcing the data value
518 #If the buffer length is lower than n then stakcing the data value
518 if self.__profIndex < self.n:
519 if self.__profIndex < self.n:
519 self.__buffer = numpy.vstack((self.__buffer, data))
520 self.__buffer = numpy.vstack((self.__buffer, data))
520 self.__profIndex += 1
521 self.__profIndex += 1
521 return
522 return
522
523
523 #If the buffer length is equal to n then replacing the last buffer value with the data value
524 #If the buffer length is equal to n then replacing the last buffer value with the data value
524 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
525 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
525 self.__buffer[self.n-1] = data
526 self.__buffer[self.n-1] = data
526 self.__profIndex = self.n
527 self.__profIndex = self.n
527 return
528 return
528
529
529
530
530 def pushData(self):
531 def pushData(self):
531 """
532 """
532 Return the sum of the last profiles and the profiles used in the sum.
533 Return the sum of the last profiles and the profiles used in the sum.
533
534
534 Affected:
535 Affected:
535
536
536 self.__profileIndex
537 self.__profileIndex
537
538
538 """
539 """
539
540
540 if not self.__withOverlapping:
541 if not self.__withOverlapping:
541 data = self.__buffer
542 data = self.__buffer
542 n = self.__profIndex
543 n = self.__profIndex
543
544
544 self.__buffer = 0
545 self.__buffer = 0
545 self.__profIndex = 0
546 self.__profIndex = 0
546
547
547 return data, n
548 return data, n
548
549
549 #Integration with Overlapping
550 #Integration with Overlapping
550 data = numpy.sum(self.__buffer, axis=0)
551 data = numpy.sum(self.__buffer, axis=0)
551 # print data
552 # print data
552 # raise
553 # raise
553 n = self.__profIndex
554 n = self.__profIndex
554
555
555 return data, n
556 return data, n
556
557
557 def byProfiles(self, data):
558 def byProfiles(self, data):
558
559
559 self.__dataReady = False
560 self.__dataReady = False
560 avgdata = None
561 avgdata = None
561 # n = None
562 # n = None
562 # print data
563 # print data
563 # raise
564 # raise
564 self.putData(data)
565 self.putData(data)
565
566
566 if self.__profIndex == self.n:
567 if self.__profIndex == self.n:
567 avgdata, n = self.pushData()
568 avgdata, n = self.pushData()
568 self.__dataReady = True
569 self.__dataReady = True
569
570
570 return avgdata
571 return avgdata
571
572
572 def byTime(self, data, datatime):
573 def byTime(self, data, datatime):
573
574
574 self.__dataReady = False
575 self.__dataReady = False
575 avgdata = None
576 avgdata = None
576 n = None
577 n = None
577
578
578 self.putData(data)
579 self.putData(data)
579
580
580 if (datatime - self.__initime) >= self.__integrationtime:
581 if (datatime - self.__initime) >= self.__integrationtime:
581 avgdata, n = self.pushData()
582 avgdata, n = self.pushData()
582 self.n = n
583 self.n = n
583 self.__dataReady = True
584 self.__dataReady = True
584
585
585 return avgdata
586 return avgdata
586
587
587 def integrateByStride(self, data, datatime):
588 def integrateByStride(self, data, datatime):
588 # print data
589 # print data
589 if self.__profIndex == 0:
590 if self.__profIndex == 0:
590 self.__buffer = [[data.copy(), datatime]]
591 self.__buffer = [[data.copy(), datatime]]
591 else:
592 else:
592 self.__buffer.append([data.copy(),datatime])
593 self.__buffer.append([data.copy(),datatime])
593 self.__profIndex += 1
594 self.__profIndex += 1
594 self.__dataReady = False
595 self.__dataReady = False
595
596
596 if self.__profIndex == self.n * self.stride :
597 if self.__profIndex == self.n * self.stride :
597 self.__dataToPutStride = True
598 self.__dataToPutStride = True
598 self.__profIndexStride = 0
599 self.__profIndexStride = 0
599 self.__profIndex = 0
600 self.__profIndex = 0
600 self.__bufferStride = []
601 self.__bufferStride = []
601 for i in range(self.stride):
602 for i in range(self.stride):
602 current = self.__buffer[i::self.stride]
603 current = self.__buffer[i::self.stride]
603 data = numpy.sum([t[0] for t in current], axis=0)
604 data = numpy.sum([t[0] for t in current], axis=0)
604 avgdatatime = numpy.average([t[1] for t in current])
605 avgdatatime = numpy.average([t[1] for t in current])
605 # print data
606 # print data
606 self.__bufferStride.append((data, avgdatatime))
607 self.__bufferStride.append((data, avgdatatime))
607
608
608 if self.__dataToPutStride:
609 if self.__dataToPutStride:
609 self.__dataReady = True
610 self.__dataReady = True
610 self.__profIndexStride += 1
611 self.__profIndexStride += 1
611 if self.__profIndexStride == self.stride:
612 if self.__profIndexStride == self.stride:
612 self.__dataToPutStride = False
613 self.__dataToPutStride = False
613 # print self.__bufferStride[self.__profIndexStride - 1]
614 # print self.__bufferStride[self.__profIndexStride - 1]
614 # raise
615 # raise
615 return self.__bufferStride[self.__profIndexStride - 1]
616 return self.__bufferStride[self.__profIndexStride - 1]
616
617
617
618
618 return None, None
619 return None, None
619
620
620 def integrate(self, data, datatime=None):
621 def integrate(self, data, datatime=None):
621
622
622 if self.__initime == None:
623 if self.__initime == None:
623 self.__initime = datatime
624 self.__initime = datatime
624
625
625 if self.__byTime:
626 if self.__byTime:
626 avgdata = self.byTime(data, datatime)
627 avgdata = self.byTime(data, datatime)
627 else:
628 else:
628 avgdata = self.byProfiles(data)
629 avgdata = self.byProfiles(data)
629
630
630
631
631 self.__lastdatatime = datatime
632 self.__lastdatatime = datatime
632
633
633 if avgdata is None:
634 if avgdata is None:
634 return None, None
635 return None, None
635
636
636 avgdatatime = self.__initime
637 avgdatatime = self.__initime
637
638
638 deltatime = datatime - self.__lastdatatime
639 deltatime = datatime - self.__lastdatatime
639
640
640 if not self.__withOverlapping:
641 if not self.__withOverlapping:
641 self.__initime = datatime
642 self.__initime = datatime
642 else:
643 else:
643 self.__initime += deltatime
644 self.__initime += deltatime
644
645
645 return avgdata, avgdatatime
646 return avgdata, avgdatatime
646
647
647 def integrateByBlock(self, dataOut):
648 def integrateByBlock(self, dataOut):
648
649
649 times = int(dataOut.data.shape[1]/self.n)
650 times = int(dataOut.data.shape[1]/self.n)
650 avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
651 avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
651
652
652 id_min = 0
653 id_min = 0
653 id_max = self.n
654 id_max = self.n
654
655
655 for i in range(times):
656 for i in range(times):
656 junk = dataOut.data[:,id_min:id_max,:]
657 junk = dataOut.data[:,id_min:id_max,:]
657 avgdata[:,i,:] = junk.sum(axis=1)
658 avgdata[:,i,:] = junk.sum(axis=1)
658 id_min += self.n
659 id_min += self.n
659 id_max += self.n
660 id_max += self.n
660
661
661 timeInterval = dataOut.ippSeconds*self.n
662 timeInterval = dataOut.ippSeconds*self.n
662 avgdatatime = (times - 1) * timeInterval + dataOut.utctime
663 avgdatatime = (times - 1) * timeInterval + dataOut.utctime
663 self.__dataReady = True
664 self.__dataReady = True
664 return avgdata, avgdatatime
665 return avgdata, avgdatatime
665
666
666 def run(self, dataOut, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False, **kwargs):
667 def run(self, dataOut, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False, **kwargs):
667
668
668 if not self.isConfig:
669 if not self.isConfig:
669 self.setup(n=n, stride=stride, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
670 self.setup(n=n, stride=stride, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
670 self.isConfig = True
671 self.isConfig = True
671
672
672 if dataOut.flagDataAsBlock:
673 if dataOut.flagDataAsBlock:
673 """
674 """
674 Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
675 Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
675 """
676 """
676 avgdata, avgdatatime = self.integrateByBlock(dataOut)
677 avgdata, avgdatatime = self.integrateByBlock(dataOut)
677 dataOut.nProfiles /= self.n
678 dataOut.nProfiles /= self.n
678 else:
679 else:
679 if stride is None:
680 if stride is None:
680 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
681 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
681 else:
682 else:
682 avgdata, avgdatatime = self.integrateByStride(dataOut.data, dataOut.utctime)
683 avgdata, avgdatatime = self.integrateByStride(dataOut.data, dataOut.utctime)
683
684
684
685
685 # dataOut.timeInterval *= n
686 # dataOut.timeInterval *= n
686 dataOut.flagNoData = True
687 dataOut.flagNoData = True
687
688
688 if self.__dataReady:
689 if self.__dataReady:
689 dataOut.data = avgdata
690 dataOut.data = avgdata
690 if not dataOut.flagCohInt:
691 if not dataOut.flagCohInt:
691 dataOut.nCohInt *= self.n
692 dataOut.nCohInt *= self.n
692 dataOut.flagCohInt = True
693 dataOut.flagCohInt = True
693 dataOut.utctime = avgdatatime
694 dataOut.utctime = avgdatatime
694 # print avgdata, avgdatatime
695 # print avgdata, avgdatatime
695 # raise
696 # raise
696 # dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
697 # dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
697 dataOut.flagNoData = False
698 dataOut.flagNoData = False
698 return dataOut
699 return dataOut
699
700
700 class Decoder(Operation):
701 class Decoder(Operation):
701
702
702 isConfig = False
703 isConfig = False
703 __profIndex = 0
704 __profIndex = 0
704
705
705 code = None
706 code = None
706
707
707 nCode = None
708 nCode = None
708 nBaud = None
709 nBaud = None
709
710
710 def __init__(self, **kwargs):
711 def __init__(self, **kwargs):
711
712
712 Operation.__init__(self, **kwargs)
713 Operation.__init__(self, **kwargs)
713
714
714 self.times = None
715 self.times = None
715 self.osamp = None
716 self.osamp = None
716 # self.__setValues = False
717 # self.__setValues = False
717 self.isConfig = False
718 self.isConfig = False
718 self.setupReq = False
719 self.setupReq = False
719 def setup(self, code, osamp, dataOut):
720 def setup(self, code, osamp, dataOut):
720
721
721 self.__profIndex = 0
722 self.__profIndex = 0
722
723
723 self.code = code
724 self.code = code
724
725
725 self.nCode = len(code)
726 self.nCode = len(code)
726 self.nBaud = len(code[0])
727 self.nBaud = len(code[0])
727 if (osamp != None) and (osamp >1):
728 if (osamp != None) and (osamp >1):
728 self.osamp = osamp
729 self.osamp = osamp
729 self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
730 self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
730 self.nBaud = self.nBaud*self.osamp
731 self.nBaud = self.nBaud*self.osamp
731
732
732 self.__nChannels = dataOut.nChannels
733 self.__nChannels = dataOut.nChannels
733 self.__nProfiles = dataOut.nProfiles
734 self.__nProfiles = dataOut.nProfiles
734 self.__nHeis = dataOut.nHeights
735 self.__nHeis = dataOut.nHeights
735
736
736 if self.__nHeis < self.nBaud:
737 if self.__nHeis < self.nBaud:
737 raise ValueError('Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud))
738 raise ValueError('Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud))
738
739
739 #Frequency
740 #Frequency
740 __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
741 __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
741
742
742 __codeBuffer[:,0:self.nBaud] = self.code
743 __codeBuffer[:,0:self.nBaud] = self.code
743
744
744 self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
745 self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
745
746
746 if dataOut.flagDataAsBlock:
747 if dataOut.flagDataAsBlock:
747
748
748 self.ndatadec = self.__nHeis #- self.nBaud + 1
749 self.ndatadec = self.__nHeis #- self.nBaud + 1
749
750
750 self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
751 self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
751
752
752 else:
753 else:
753
754
754 #Time
755 #Time
755 self.ndatadec = self.__nHeis #- self.nBaud + 1
756 self.ndatadec = self.__nHeis #- self.nBaud + 1
756
757
757 self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
758 self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
758
759
759 def __convolutionInFreq(self, data):
760 def __convolutionInFreq(self, data):
760
761
761 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
762 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
762
763
763 fft_data = numpy.fft.fft(data, axis=1)
764 fft_data = numpy.fft.fft(data, axis=1)
764
765
765 conv = fft_data*fft_code
766 conv = fft_data*fft_code
766
767
767 data = numpy.fft.ifft(conv,axis=1)
768 data = numpy.fft.ifft(conv,axis=1)
768
769
769 return data
770 return data
770
771
771 def __convolutionInFreqOpt(self, data):
772 def __convolutionInFreqOpt(self, data):
772
773
773 raise NotImplementedError
774 raise NotImplementedError
774
775
775 def __convolutionInTime(self, data):
776 def __convolutionInTime(self, data):
776
777
777 code = self.code[self.__profIndex]
778 code = self.code[self.__profIndex]
778 for i in range(self.__nChannels):
779 for i in range(self.__nChannels):
779 self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
780 self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
780
781
781 return self.datadecTime
782 return self.datadecTime
782
783
783 def __convolutionByBlockInTime(self, data):
784 def __convolutionByBlockInTime(self, data):
784
785
785 repetitions = int(self.__nProfiles / self.nCode)
786 repetitions = int(self.__nProfiles / self.nCode)
786 junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
787 junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
787 junk = junk.flatten()
788 junk = junk.flatten()
788 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
789 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
789 profilesList = range(self.__nProfiles)
790 profilesList = range(self.__nProfiles)
790
791
791 for i in range(self.__nChannels):
792 for i in range(self.__nChannels):
792 for j in profilesList:
793 for j in profilesList:
793 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
794 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
794 return self.datadecTime
795 return self.datadecTime
795
796
796 def __convolutionByBlockInFreq(self, data):
797 def __convolutionByBlockInFreq(self, data):
797
798
798 raise NotImplementedError("Decoder by frequency fro Blocks not implemented")
799 raise NotImplementedError("Decoder by frequency fro Blocks not implemented")
799
800
800
801
801 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
802 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
802
803
803 fft_data = numpy.fft.fft(data, axis=2)
804 fft_data = numpy.fft.fft(data, axis=2)
804
805
805 conv = fft_data*fft_code
806 conv = fft_data*fft_code
806
807
807 data = numpy.fft.ifft(conv,axis=2)
808 data = numpy.fft.ifft(conv,axis=2)
808
809
809 return data
810 return data
810
811
811
812
812 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
813 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
813
814
814 if dataOut.flagDecodeData:
815 if dataOut.flagDecodeData:
815 print("This data is already decoded, recoding again ...")
816 print("This data is already decoded, recoding again ...")
816
817
817 if not self.isConfig:
818 if not self.isConfig:
818
819
819 if code is None:
820 if code is None:
820 if dataOut.code is None:
821 if dataOut.code is None:
821 raise ValueError("Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type)
822 raise ValueError("Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type)
822
823
823 code = dataOut.code
824 code = dataOut.code
824 else:
825 else:
825 code = numpy.array(code).reshape(nCode,nBaud)
826 code = numpy.array(code).reshape(nCode,nBaud)
826 self.setup(code, osamp, dataOut)
827 self.setup(code, osamp, dataOut)
827
828
828 self.isConfig = True
829 self.isConfig = True
829
830
830 if mode == 3:
831 if mode == 3:
831 sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
832 sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
832
833
833 if times != None:
834 if times != None:
834 sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
835 sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
835
836
836 if self.code is None:
837 if self.code is None:
837 print("Fail decoding: Code is not defined.")
838 print("Fail decoding: Code is not defined.")
838 return
839 return
839
840
840 self.__nProfiles = dataOut.nProfiles
841 self.__nProfiles = dataOut.nProfiles
841 datadec = None
842 datadec = None
842
843
843 if mode == 3:
844 if mode == 3:
844 mode = 0
845 mode = 0
845
846
846 if dataOut.flagDataAsBlock:
847 if dataOut.flagDataAsBlock:
847 """
848 """
848 Decoding when data have been read as block,
849 Decoding when data have been read as block,
849 """
850 """
850
851
851 if mode == 0:
852 if mode == 0:
852 datadec = self.__convolutionByBlockInTime(dataOut.data)
853 datadec = self.__convolutionByBlockInTime(dataOut.data)
853 if mode == 1:
854 if mode == 1:
854 datadec = self.__convolutionByBlockInFreq(dataOut.data)
855 datadec = self.__convolutionByBlockInFreq(dataOut.data)
855 else:
856 else:
856 """
857 """
857 Decoding when data have been read profile by profile
858 Decoding when data have been read profile by profile
858 """
859 """
859 if mode == 0:
860 if mode == 0:
860 datadec = self.__convolutionInTime(dataOut.data)
861 datadec = self.__convolutionInTime(dataOut.data)
861
862
862 if mode == 1:
863 if mode == 1:
863 datadec = self.__convolutionInFreq(dataOut.data)
864 datadec = self.__convolutionInFreq(dataOut.data)
864
865
865 if mode == 2:
866 if mode == 2:
866 datadec = self.__convolutionInFreqOpt(dataOut.data)
867 datadec = self.__convolutionInFreqOpt(dataOut.data)
867
868
868 if datadec is None:
869 if datadec is None:
869 raise ValueError("Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode)
870 raise ValueError("Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode)
870
871
871 dataOut.code = self.code
872 dataOut.code = self.code
872 dataOut.nCode = self.nCode
873 dataOut.nCode = self.nCode
873 dataOut.nBaud = self.nBaud
874 dataOut.nBaud = self.nBaud
874
875
875 dataOut.data = datadec
876 dataOut.data = datadec
876
877
877 dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
878 dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
878
879
879 dataOut.flagDecodeData = True #asumo q la data esta decodificada
880 dataOut.flagDecodeData = True #asumo q la data esta decodificada
880
881
881 if self.__profIndex == self.nCode-1:
882 if self.__profIndex == self.nCode-1:
882 self.__profIndex = 0
883 self.__profIndex = 0
883 return dataOut
884 return dataOut
884
885
885 self.__profIndex += 1
886 self.__profIndex += 1
886
887
887 return dataOut
888 return dataOut
888 # dataOut.flagDeflipData = True #asumo q la data no esta sin flip
889 # dataOut.flagDeflipData = True #asumo q la data no esta sin flip
889
890
890
891
891 class ProfileConcat(Operation):
892 class ProfileConcat(Operation):
892
893
893 isConfig = False
894 isConfig = False
894 buffer = None
895 buffer = None
895
896
896 def __init__(self, **kwargs):
897 def __init__(self, **kwargs):
897
898
898 Operation.__init__(self, **kwargs)
899 Operation.__init__(self, **kwargs)
899 self.profileIndex = 0
900 self.profileIndex = 0
900
901
901 def reset(self):
902 def reset(self):
902 self.buffer = numpy.zeros_like(self.buffer)
903 self.buffer = numpy.zeros_like(self.buffer)
903 self.start_index = 0
904 self.start_index = 0
904 self.times = 1
905 self.times = 1
905
906
906 def setup(self, data, m, n=1):
907 def setup(self, data, m, n=1):
907 self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
908 self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
908 self.nHeights = data.shape[1]#.nHeights
909 self.nHeights = data.shape[1]#.nHeights
909 self.start_index = 0
910 self.start_index = 0
910 self.times = 1
911 self.times = 1
911
912
912 def concat(self, data):
913 def concat(self, data):
913
914
914 self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
915 self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
915 self.start_index = self.start_index + self.nHeights
916 self.start_index = self.start_index + self.nHeights
916
917
917 def run(self, dataOut, m):
918 def run(self, dataOut, m):
918 dataOut.flagNoData = True
919 dataOut.flagNoData = True
919
920
920 if not self.isConfig:
921 if not self.isConfig:
921 self.setup(dataOut.data, m, 1)
922 self.setup(dataOut.data, m, 1)
922 self.isConfig = True
923 self.isConfig = True
923
924
924 if dataOut.flagDataAsBlock:
925 if dataOut.flagDataAsBlock:
925 raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
926 raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
926
927
927 else:
928 else:
928 self.concat(dataOut.data)
929 self.concat(dataOut.data)
929 self.times += 1
930 self.times += 1
930 if self.times > m:
931 if self.times > m:
931 dataOut.data = self.buffer
932 dataOut.data = self.buffer
932 self.reset()
933 self.reset()
933 dataOut.flagNoData = False
934 dataOut.flagNoData = False
934 # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
935 # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
935 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
936 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
936 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
937 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
937 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
938 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
938 dataOut.ippSeconds *= m
939 dataOut.ippSeconds *= m
939 return dataOut
940 return dataOut
940
941
941 class ProfileSelector(Operation):
942 class ProfileSelector(Operation):
942
943
943 profileIndex = None
944 profileIndex = None
944 # Tamanho total de los perfiles
945 # Tamanho total de los perfiles
945 nProfiles = None
946 nProfiles = None
946
947
947 def __init__(self, **kwargs):
948 def __init__(self, **kwargs):
948
949
949 Operation.__init__(self, **kwargs)
950 Operation.__init__(self, **kwargs)
950 self.profileIndex = 0
951 self.profileIndex = 0
951
952
952 def incProfileIndex(self):
953 def incProfileIndex(self):
953
954
954 self.profileIndex += 1
955 self.profileIndex += 1
955
956
956 if self.profileIndex >= self.nProfiles:
957 if self.profileIndex >= self.nProfiles:
957 self.profileIndex = 0
958 self.profileIndex = 0
958
959
959 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
960 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
960
961
961 if profileIndex < minIndex:
962 if profileIndex < minIndex:
962 return False
963 return False
963
964
964 if profileIndex > maxIndex:
965 if profileIndex > maxIndex:
965 return False
966 return False
966
967
967 return True
968 return True
968
969
969 def isThisProfileInList(self, profileIndex, profileList):
970 def isThisProfileInList(self, profileIndex, profileList):
970
971
971 if profileIndex not in profileList:
972 if profileIndex not in profileList:
972 return False
973 return False
973
974
974 return True
975 return True
975
976
976 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
977 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
977
978
978 """
979 """
979 ProfileSelector:
980 ProfileSelector:
980
981
981 Inputs:
982 Inputs:
982 profileList : Index of profiles selected. Example: profileList = (0,1,2,7,8)
983 profileList : Index of profiles selected. Example: profileList = (0,1,2,7,8)
983
984
984 profileRangeList : Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
985 profileRangeList : Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
985
986
986 rangeList : List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
987 rangeList : List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
987
988
988 """
989 """
989
990
990 if rangeList is not None:
991 if rangeList is not None:
991 if type(rangeList[0]) not in (tuple, list):
992 if type(rangeList[0]) not in (tuple, list):
992 rangeList = [rangeList]
993 rangeList = [rangeList]
993
994
994 dataOut.flagNoData = True
995 dataOut.flagNoData = True
995
996
996 if dataOut.flagDataAsBlock:
997 if dataOut.flagDataAsBlock:
997 """
998 """
998 data dimension = [nChannels, nProfiles, nHeis]
999 data dimension = [nChannels, nProfiles, nHeis]
999 """
1000 """
1000 if profileList != None:
1001 if profileList != None:
1001 dataOut.data = dataOut.data[:,profileList,:]
1002 dataOut.data = dataOut.data[:,profileList,:]
1002
1003
1003 if profileRangeList != None:
1004 if profileRangeList != None:
1004 minIndex = profileRangeList[0]
1005 minIndex = profileRangeList[0]
1005 maxIndex = profileRangeList[1]
1006 maxIndex = profileRangeList[1]
1006 profileList = list(range(minIndex, maxIndex+1))
1007 profileList = list(range(minIndex, maxIndex+1))
1007
1008
1008 dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
1009 dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
1009
1010
1010 if rangeList != None:
1011 if rangeList != None:
1011
1012
1012 profileList = []
1013 profileList = []
1013
1014
1014 for thisRange in rangeList:
1015 for thisRange in rangeList:
1015 minIndex = thisRange[0]
1016 minIndex = thisRange[0]
1016 maxIndex = thisRange[1]
1017 maxIndex = thisRange[1]
1017
1018
1018 profileList.extend(list(range(minIndex, maxIndex+1)))
1019 profileList.extend(list(range(minIndex, maxIndex+1)))
1019
1020
1020 dataOut.data = dataOut.data[:,profileList,:]
1021 dataOut.data = dataOut.data[:,profileList,:]
1021
1022
1022 dataOut.nProfiles = len(profileList)
1023 dataOut.nProfiles = len(profileList)
1023 dataOut.profileIndex = dataOut.nProfiles - 1
1024 dataOut.profileIndex = dataOut.nProfiles - 1
1024 dataOut.flagNoData = False
1025 dataOut.flagNoData = False
1025
1026
1026 return dataOut
1027 return dataOut
1027
1028
1028 """
1029 """
1029 data dimension = [nChannels, nHeis]
1030 data dimension = [nChannels, nHeis]
1030 """
1031 """
1031
1032
1032 if profileList != None:
1033 if profileList != None:
1033
1034
1034 if self.isThisProfileInList(dataOut.profileIndex, profileList):
1035 if self.isThisProfileInList(dataOut.profileIndex, profileList):
1035
1036
1036 self.nProfiles = len(profileList)
1037 self.nProfiles = len(profileList)
1037 dataOut.nProfiles = self.nProfiles
1038 dataOut.nProfiles = self.nProfiles
1038 dataOut.profileIndex = self.profileIndex
1039 dataOut.profileIndex = self.profileIndex
1039 dataOut.flagNoData = False
1040 dataOut.flagNoData = False
1040
1041
1041 self.incProfileIndex()
1042 self.incProfileIndex()
1042 return dataOut
1043 return dataOut
1043
1044
1044 if profileRangeList != None:
1045 if profileRangeList != None:
1045
1046
1046 minIndex = profileRangeList[0]
1047 minIndex = profileRangeList[0]
1047 maxIndex = profileRangeList[1]
1048 maxIndex = profileRangeList[1]
1048
1049
1049 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
1050 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
1050
1051
1051 self.nProfiles = maxIndex - minIndex + 1
1052 self.nProfiles = maxIndex - minIndex + 1
1052 dataOut.nProfiles = self.nProfiles
1053 dataOut.nProfiles = self.nProfiles
1053 dataOut.profileIndex = self.profileIndex
1054 dataOut.profileIndex = self.profileIndex
1054 dataOut.flagNoData = False
1055 dataOut.flagNoData = False
1055
1056
1056 self.incProfileIndex()
1057 self.incProfileIndex()
1057 return dataOut
1058 return dataOut
1058
1059
1059 if rangeList != None:
1060 if rangeList != None:
1060
1061
1061 nProfiles = 0
1062 nProfiles = 0
1062
1063
1063 for thisRange in rangeList:
1064 for thisRange in rangeList:
1064 minIndex = thisRange[0]
1065 minIndex = thisRange[0]
1065 maxIndex = thisRange[1]
1066 maxIndex = thisRange[1]
1066
1067
1067 nProfiles += maxIndex - minIndex + 1
1068 nProfiles += maxIndex - minIndex + 1
1068
1069
1069 for thisRange in rangeList:
1070 for thisRange in rangeList:
1070
1071
1071 minIndex = thisRange[0]
1072 minIndex = thisRange[0]
1072 maxIndex = thisRange[1]
1073 maxIndex = thisRange[1]
1073
1074
1074 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
1075 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
1075
1076
1076 self.nProfiles = nProfiles
1077 self.nProfiles = nProfiles
1077 dataOut.nProfiles = self.nProfiles
1078 dataOut.nProfiles = self.nProfiles
1078 dataOut.profileIndex = self.profileIndex
1079 dataOut.profileIndex = self.profileIndex
1079 dataOut.flagNoData = False
1080 dataOut.flagNoData = False
1080
1081
1081 self.incProfileIndex()
1082 self.incProfileIndex()
1082
1083
1083 break
1084 break
1084
1085
1085 return dataOut
1086 return dataOut
1086
1087
1087
1088
1088 if beam != None: #beam is only for AMISR data
1089 if beam != None: #beam is only for AMISR data
1089 if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
1090 if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
1090 dataOut.flagNoData = False
1091 dataOut.flagNoData = False
1091 dataOut.profileIndex = self.profileIndex
1092 dataOut.profileIndex = self.profileIndex
1092
1093
1093 self.incProfileIndex()
1094 self.incProfileIndex()
1094
1095
1095 return dataOut
1096 return dataOut
1096
1097
1097 raise ValueError("ProfileSelector needs profileList, profileRangeList or rangeList parameter")
1098 raise ValueError("ProfileSelector needs profileList, profileRangeList or rangeList parameter")
1098
1099
1099
1100
1100 class Reshaper(Operation):
1101 class Reshaper(Operation):
1101
1102
1102 def __init__(self, **kwargs):
1103 def __init__(self, **kwargs):
1103
1104
1104 Operation.__init__(self, **kwargs)
1105 Operation.__init__(self, **kwargs)
1105
1106
1106 self.__buffer = None
1107 self.__buffer = None
1107 self.__nitems = 0
1108 self.__nitems = 0
1108
1109
1109 def __appendProfile(self, dataOut, nTxs):
1110 def __appendProfile(self, dataOut, nTxs):
1110
1111
1111 if self.__buffer is None:
1112 if self.__buffer is None:
1112 shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
1113 shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
1113 self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
1114 self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
1114
1115
1115 ini = dataOut.nHeights * self.__nitems
1116 ini = dataOut.nHeights * self.__nitems
1116 end = ini + dataOut.nHeights
1117 end = ini + dataOut.nHeights
1117
1118
1118 self.__buffer[:, ini:end] = dataOut.data
1119 self.__buffer[:, ini:end] = dataOut.data
1119
1120
1120 self.__nitems += 1
1121 self.__nitems += 1
1121
1122
1122 return int(self.__nitems*nTxs)
1123 return int(self.__nitems*nTxs)
1123
1124
1124 def __getBuffer(self):
1125 def __getBuffer(self):
1125
1126
1126 if self.__nitems == int(1./self.__nTxs):
1127 if self.__nitems == int(1./self.__nTxs):
1127
1128
1128 self.__nitems = 0
1129 self.__nitems = 0
1129
1130
1130 return self.__buffer.copy()
1131 return self.__buffer.copy()
1131
1132
1132 return None
1133 return None
1133
1134
1134 def __checkInputs(self, dataOut, shape, nTxs):
1135 def __checkInputs(self, dataOut, shape, nTxs):
1135
1136
1136 if shape is None and nTxs is None:
1137 if shape is None and nTxs is None:
1137 raise ValueError("Reshaper: shape of factor should be defined")
1138 raise ValueError("Reshaper: shape of factor should be defined")
1138
1139
1139 if nTxs:
1140 if nTxs:
1140 if nTxs < 0:
1141 if nTxs < 0:
1141 raise ValueError("nTxs should be greater than 0")
1142 raise ValueError("nTxs should be greater than 0")
1142
1143
1143 if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
1144 if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
1144 raise ValueError("nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs)))
1145 raise ValueError("nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs)))
1145
1146
1146 shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
1147 shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
1147
1148
1148 return shape, nTxs
1149 return shape, nTxs
1149
1150
1150 if len(shape) != 2 and len(shape) != 3:
1151 if len(shape) != 2 and len(shape) != 3:
1151 raise ValueError("shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights))
1152 raise ValueError("shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights))
1152
1153
1153 if len(shape) == 2:
1154 if len(shape) == 2:
1154 shape_tuple = [dataOut.nChannels]
1155 shape_tuple = [dataOut.nChannels]
1155 shape_tuple.extend(shape)
1156 shape_tuple.extend(shape)
1156 else:
1157 else:
1157 shape_tuple = list(shape)
1158 shape_tuple = list(shape)
1158
1159
1159 nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
1160 nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
1160
1161
1161 return shape_tuple, nTxs
1162 return shape_tuple, nTxs
1162
1163
1163 def run(self, dataOut, shape=None, nTxs=None):
1164 def run(self, dataOut, shape=None, nTxs=None):
1164
1165
1165 shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
1166 shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
1166
1167
1167 dataOut.flagNoData = True
1168 dataOut.flagNoData = True
1168 profileIndex = None
1169 profileIndex = None
1169
1170
1170 if dataOut.flagDataAsBlock:
1171 if dataOut.flagDataAsBlock:
1171
1172
1172 dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
1173 dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
1173 dataOut.flagNoData = False
1174 dataOut.flagNoData = False
1174
1175
1175 profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
1176 profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
1176
1177
1177 else:
1178 else:
1178
1179
1179 if self.__nTxs < 1:
1180 if self.__nTxs < 1:
1180
1181
1181 self.__appendProfile(dataOut, self.__nTxs)
1182 self.__appendProfile(dataOut, self.__nTxs)
1182 new_data = self.__getBuffer()
1183 new_data = self.__getBuffer()
1183
1184
1184 if new_data is not None:
1185 if new_data is not None:
1185 dataOut.data = new_data
1186 dataOut.data = new_data
1186 dataOut.flagNoData = False
1187 dataOut.flagNoData = False
1187
1188
1188 profileIndex = dataOut.profileIndex*nTxs
1189 profileIndex = dataOut.profileIndex*nTxs
1189
1190
1190 else:
1191 else:
1191 raise ValueError("nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)")
1192 raise ValueError("nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)")
1192
1193
1193 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1194 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1194
1195
1195 dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
1196 dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
1196
1197
1197 dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
1198 dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
1198
1199
1199 dataOut.profileIndex = profileIndex
1200 dataOut.profileIndex = profileIndex
1200
1201
1201 dataOut.ippSeconds /= self.__nTxs
1202 dataOut.ippSeconds /= self.__nTxs
1202
1203
1203 return dataOut
1204 return dataOut
1204
1205
1205 class SplitProfiles(Operation):
1206 class SplitProfiles(Operation):
1206
1207
1207 def __init__(self, **kwargs):
1208 def __init__(self, **kwargs):
1208
1209
1209 Operation.__init__(self, **kwargs)
1210 Operation.__init__(self, **kwargs)
1210
1211
1211 def run(self, dataOut, n):
1212 def run(self, dataOut, n):
1212
1213
1213 dataOut.flagNoData = True
1214 dataOut.flagNoData = True
1214 profileIndex = None
1215 profileIndex = None
1215
1216
1216 if dataOut.flagDataAsBlock:
1217 if dataOut.flagDataAsBlock:
1217
1218
1218 #nchannels, nprofiles, nsamples
1219 #nchannels, nprofiles, nsamples
1219 shape = dataOut.data.shape
1220 shape = dataOut.data.shape
1220
1221
1221 if shape[2] % n != 0:
1222 if shape[2] % n != 0:
1222 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[2]))
1223 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[2]))
1223
1224
1224 new_shape = shape[0], shape[1]*n, int(shape[2]/n)
1225 new_shape = shape[0], shape[1]*n, int(shape[2]/n)
1225
1226
1226 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1227 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1227 dataOut.flagNoData = False
1228 dataOut.flagNoData = False
1228
1229
1229 profileIndex = int(dataOut.nProfiles/n) - 1
1230 profileIndex = int(dataOut.nProfiles/n) - 1
1230
1231
1231 else:
1232 else:
1232
1233
1233 raise ValueError("Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)")
1234 raise ValueError("Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)")
1234
1235
1235 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1236 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1236
1237
1237 dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
1238 dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
1238
1239
1239 dataOut.nProfiles = int(dataOut.nProfiles*n)
1240 dataOut.nProfiles = int(dataOut.nProfiles*n)
1240
1241
1241 dataOut.profileIndex = profileIndex
1242 dataOut.profileIndex = profileIndex
1242
1243
1243 dataOut.ippSeconds /= n
1244 dataOut.ippSeconds /= n
1244
1245
1245 return dataOut
1246 return dataOut
1246
1247
1247 class CombineProfiles(Operation):
1248 class CombineProfiles(Operation):
1248 def __init__(self, **kwargs):
1249 def __init__(self, **kwargs):
1249
1250
1250 Operation.__init__(self, **kwargs)
1251 Operation.__init__(self, **kwargs)
1251
1252
1252 self.__remData = None
1253 self.__remData = None
1253 self.__profileIndex = 0
1254 self.__profileIndex = 0
1254
1255
1255 def run(self, dataOut, n):
1256 def run(self, dataOut, n):
1256
1257
1257 dataOut.flagNoData = True
1258 dataOut.flagNoData = True
1258 profileIndex = None
1259 profileIndex = None
1259
1260
1260 if dataOut.flagDataAsBlock:
1261 if dataOut.flagDataAsBlock:
1261
1262
1262 #nchannels, nprofiles, nsamples
1263 #nchannels, nprofiles, nsamples
1263 shape = dataOut.data.shape
1264 shape = dataOut.data.shape
1264 new_shape = shape[0], shape[1]/n, shape[2]*n
1265 new_shape = shape[0], shape[1]/n, shape[2]*n
1265
1266
1266 if shape[1] % n != 0:
1267 if shape[1] % n != 0:
1267 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[1]))
1268 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[1]))
1268
1269
1269 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1270 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1270 dataOut.flagNoData = False
1271 dataOut.flagNoData = False
1271
1272
1272 profileIndex = int(dataOut.nProfiles*n) - 1
1273 profileIndex = int(dataOut.nProfiles*n) - 1
1273
1274
1274 else:
1275 else:
1275
1276
1276 #nchannels, nsamples
1277 #nchannels, nsamples
1277 if self.__remData is None:
1278 if self.__remData is None:
1278 newData = dataOut.data
1279 newData = dataOut.data
1279 else:
1280 else:
1280 newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
1281 newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
1281
1282
1282 self.__profileIndex += 1
1283 self.__profileIndex += 1
1283
1284
1284 if self.__profileIndex < n:
1285 if self.__profileIndex < n:
1285 self.__remData = newData
1286 self.__remData = newData
1286 #continue
1287 #continue
1287 return
1288 return
1288
1289
1289 self.__profileIndex = 0
1290 self.__profileIndex = 0
1290 self.__remData = None
1291 self.__remData = None
1291
1292
1292 dataOut.data = newData
1293 dataOut.data = newData
1293 dataOut.flagNoData = False
1294 dataOut.flagNoData = False
1294
1295
1295 profileIndex = dataOut.profileIndex/n
1296 profileIndex = dataOut.profileIndex/n
1296
1297
1297
1298
1298 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1299 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1299
1300
1300 dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
1301 dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
1301
1302
1302 dataOut.nProfiles = int(dataOut.nProfiles/n)
1303 dataOut.nProfiles = int(dataOut.nProfiles/n)
1303
1304
1304 dataOut.profileIndex = profileIndex
1305 dataOut.profileIndex = profileIndex
1305
1306
1306 dataOut.ippSeconds *= n
1307 dataOut.ippSeconds *= n
1307
1308
1308 return dataOut
1309 return dataOut
1309
1310
1310 class PulsePairVoltage(Operation):
1311 class PulsePairVoltage(Operation):
1311 '''
1312 '''
1312 Function PulsePair(Signal Power, Velocity)
1313 Function PulsePair(Signal Power, Velocity)
1313 The real component of Lag[0] provides Intensity Information
1314 The real component of Lag[0] provides Intensity Information
1314 The imag component of Lag[1] Phase provides Velocity Information
1315 The imag component of Lag[1] Phase provides Velocity Information
1315
1316
1316 Configuration Parameters:
1317 Configuration Parameters:
1317 nPRF = Number of Several PRF
1318 nPRF = Number of Several PRF
1318 theta = Degree Azimuth angel Boundaries
1319 theta = Degree Azimuth angel Boundaries
1319
1320
1320 Input:
1321 Input:
1321 self.dataOut
1322 self.dataOut
1322 lag[N]
1323 lag[N]
1323 Affected:
1324 Affected:
1324 self.dataOut.spc
1325 self.dataOut.spc
1325 '''
1326 '''
1326 isConfig = False
1327 isConfig = False
1327 __profIndex = 0
1328 __profIndex = 0
1328 __initime = None
1329 __initime = None
1329 __lastdatatime = None
1330 __lastdatatime = None
1330 __buffer = None
1331 __buffer = None
1331 noise = None
1332 noise = None
1332 __dataReady = False
1333 __dataReady = False
1333 n = None
1334 n = None
1334 __nch = 0
1335 __nch = 0
1335 __nHeis = 0
1336 __nHeis = 0
1336 removeDC = False
1337 removeDC = False
1337 ipp = None
1338 ipp = None
1338 lambda_ = 0
1339 lambda_ = 0
1339
1340
1340 def __init__(self,**kwargs):
1341 def __init__(self,**kwargs):
1341 Operation.__init__(self,**kwargs)
1342 Operation.__init__(self,**kwargs)
1342
1343
1343 def setup(self, dataOut, n = None, removeDC=False):
1344 def setup(self, dataOut, n = None, removeDC=False):
1344 '''
1345 '''
1345 n= Numero de PRF's de entrada
1346 n= Numero de PRF's de entrada
1346 '''
1347 '''
1347 self.__initime = None
1348 self.__initime = None
1348 self.__lastdatatime = 0
1349 self.__lastdatatime = 0
1349 self.__dataReady = False
1350 self.__dataReady = False
1350 self.__buffer = 0
1351 self.__buffer = 0
1351 self.__profIndex = 0
1352 self.__profIndex = 0
1352 self.noise = None
1353 self.noise = None
1353 self.__nch = dataOut.nChannels
1354 self.__nch = dataOut.nChannels
1354 self.__nHeis = dataOut.nHeights
1355 self.__nHeis = dataOut.nHeights
1355 self.removeDC = removeDC
1356 self.removeDC = removeDC
1356 self.lambda_ = 3.0e8/(9345.0e6)
1357 self.lambda_ = 3.0e8/(9345.0e6)
1357 self.ippSec = dataOut.ippSeconds
1358 self.ippSec = dataOut.ippSeconds
1358 self.nCohInt = dataOut.nCohInt
1359 self.nCohInt = dataOut.nCohInt
1359
1360
1360 if n == None:
1361 if n == None:
1361 raise ValueError("n should be specified.")
1362 raise ValueError("n should be specified.")
1362
1363
1363 if n != None:
1364 if n != None:
1364 if n<2:
1365 if n<2:
1365 raise ValueError("n should be greater than 2")
1366 raise ValueError("n should be greater than 2")
1366
1367
1367 self.n = n
1368 self.n = n
1368 self.__nProf = n
1369 self.__nProf = n
1369
1370
1370 self.__buffer = numpy.zeros((dataOut.nChannels,
1371 self.__buffer = numpy.zeros((dataOut.nChannels,
1371 n,
1372 n,
1372 dataOut.nHeights),
1373 dataOut.nHeights),
1373 dtype='complex')
1374 dtype='complex')
1374
1375
1375 def putData(self,data):
1376 def putData(self,data):
1376 '''
1377 '''
1377 Add a profile to he __buffer and increase in one the __profiel Index
1378 Add a profile to he __buffer and increase in one the __profiel Index
1378 '''
1379 '''
1379 self.__buffer[:,self.__profIndex,:]= data
1380 self.__buffer[:,self.__profIndex,:]= data
1380 self.__profIndex += 1
1381 self.__profIndex += 1
1381 return
1382 return
1382
1383
1383 def pushData(self,dataOut):
1384 def pushData(self,dataOut):
1384 '''
1385 '''
1385 Return the PULSEPAIR and the profiles used in the operation
1386 Return the PULSEPAIR and the profiles used in the operation
1386 Affected : self.__profileIndex
1387 Affected : self.__profileIndex
1387 '''
1388 '''
1388 #----------------- Remove DC-----------------------------------
1389 #----------------- Remove DC-----------------------------------
1389 if self.removeDC==True:
1390 if self.removeDC==True:
1390 mean = numpy.mean(self.__buffer,1)
1391 mean = numpy.mean(self.__buffer,1)
1391 tmp = mean.reshape(self.__nch,1,self.__nHeis)
1392 tmp = mean.reshape(self.__nch,1,self.__nHeis)
1392 dc= numpy.tile(tmp,[1,self.__nProf,1])
1393 dc= numpy.tile(tmp,[1,self.__nProf,1])
1393 self.__buffer = self.__buffer - dc
1394 self.__buffer = self.__buffer - dc
1394 #------------------Calculo de Potencia ------------------------
1395 #------------------Calculo de Potencia ------------------------
1395 pair0 = self.__buffer*numpy.conj(self.__buffer)
1396 pair0 = self.__buffer*numpy.conj(self.__buffer)
1396 pair0 = pair0.real
1397 pair0 = pair0.real
1397 lag_0 = numpy.sum(pair0,1)
1398 lag_0 = numpy.sum(pair0,1)
1398 #------------------Calculo de Ruido x canal--------------------
1399 #------------------Calculo de Ruido x canal--------------------
1399 self.noise = numpy.zeros(self.__nch)
1400 self.noise = numpy.zeros(self.__nch)
1400 for i in range(self.__nch):
1401 for i in range(self.__nch):
1401 daux = numpy.sort(pair0[i,:,:],axis= None)
1402 daux = numpy.sort(pair0[i,:,:],axis= None)
1402 self.noise[i]=hildebrand_sekhon( daux ,self.nCohInt)
1403 self.noise[i]=hildebrand_sekhon( daux ,self.nCohInt)
1403
1404
1404 self.noise = self.noise.reshape(self.__nch,1)
1405 self.noise = self.noise.reshape(self.__nch,1)
1405 self.noise = numpy.tile(self.noise,[1,self.__nHeis])
1406 self.noise = numpy.tile(self.noise,[1,self.__nHeis])
1406 noise_buffer = self.noise.reshape(self.__nch,1,self.__nHeis)
1407 noise_buffer = self.noise.reshape(self.__nch,1,self.__nHeis)
1407 noise_buffer = numpy.tile(noise_buffer,[1,self.__nProf,1])
1408 noise_buffer = numpy.tile(noise_buffer,[1,self.__nProf,1])
1408 #------------------ Potencia recibida= P , Potencia senal = S , Ruido= N--
1409 #------------------ Potencia recibida= P , Potencia senal = S , Ruido= N--
1409 #------------------ P= S+N ,P=lag_0/N ---------------------------------
1410 #------------------ P= S+N ,P=lag_0/N ---------------------------------
1410 #-------------------- Power --------------------------------------------------
1411 #-------------------- Power --------------------------------------------------
1411 data_power = lag_0/(self.n*self.nCohInt)
1412 data_power = lag_0/(self.n*self.nCohInt)
1412 #------------------ Senal ---------------------------------------------------
1413 #------------------ Senal ---------------------------------------------------
1413 data_intensity = pair0 - noise_buffer
1414 data_intensity = pair0 - noise_buffer
1414 data_intensity = numpy.sum(data_intensity,axis=1)*(self.n*self.nCohInt)#*self.nCohInt)
1415 data_intensity = numpy.sum(data_intensity,axis=1)*(self.n*self.nCohInt)#*self.nCohInt)
1415 #data_intensity = (lag_0-self.noise*self.n)*(self.n*self.nCohInt)
1416 #data_intensity = (lag_0-self.noise*self.n)*(self.n*self.nCohInt)
1416 for i in range(self.__nch):
1417 for i in range(self.__nch):
1417 for j in range(self.__nHeis):
1418 for j in range(self.__nHeis):
1418 if data_intensity[i][j] < 0:
1419 if data_intensity[i][j] < 0:
1419 data_intensity[i][j] = numpy.min(numpy.absolute(data_intensity[i][j]))
1420 data_intensity[i][j] = numpy.min(numpy.absolute(data_intensity[i][j]))
1420
1421
1421 #----------------- Calculo de Frecuencia y Velocidad doppler--------
1422 #----------------- Calculo de Frecuencia y Velocidad doppler--------
1422 pair1 = self.__buffer[:,:-1,:]*numpy.conjugate(self.__buffer[:,1:,:])
1423 pair1 = self.__buffer[:,:-1,:]*numpy.conjugate(self.__buffer[:,1:,:])
1423 lag_1 = numpy.sum(pair1,1)
1424 lag_1 = numpy.sum(pair1,1)
1424 data_freq = (-1/(2.0*math.pi*self.ippSec*self.nCohInt))*numpy.angle(lag_1)
1425 data_freq = (-1/(2.0*math.pi*self.ippSec*self.nCohInt))*numpy.angle(lag_1)
1425 data_velocity = (self.lambda_/2.0)*data_freq
1426 data_velocity = (self.lambda_/2.0)*data_freq
1426
1427
1427 #---------------- Potencia promedio estimada de la Senal-----------
1428 #---------------- Potencia promedio estimada de la Senal-----------
1428 lag_0 = lag_0/self.n
1429 lag_0 = lag_0/self.n
1429 S = lag_0-self.noise
1430 S = lag_0-self.noise
1430
1431
1431 #---------------- Frecuencia Doppler promedio ---------------------
1432 #---------------- Frecuencia Doppler promedio ---------------------
1432 lag_1 = lag_1/(self.n-1)
1433 lag_1 = lag_1/(self.n-1)
1433 R1 = numpy.abs(lag_1)
1434 R1 = numpy.abs(lag_1)
1434
1435
1435 #---------------- Calculo del SNR----------------------------------
1436 #---------------- Calculo del SNR----------------------------------
1436 data_snrPP = S/self.noise
1437 data_snrPP = S/self.noise
1437 for i in range(self.__nch):
1438 for i in range(self.__nch):
1438 for j in range(self.__nHeis):
1439 for j in range(self.__nHeis):
1439 if data_snrPP[i][j] < 1.e-20:
1440 if data_snrPP[i][j] < 1.e-20:
1440 data_snrPP[i][j] = 1.e-20
1441 data_snrPP[i][j] = 1.e-20
1441
1442
1442 #----------------- Calculo del ancho espectral ----------------------
1443 #----------------- Calculo del ancho espectral ----------------------
1443 L = S/R1
1444 L = S/R1
1444 L = numpy.where(L<0,1,L)
1445 L = numpy.where(L<0,1,L)
1445 L = numpy.log(L)
1446 L = numpy.log(L)
1446 tmp = numpy.sqrt(numpy.absolute(L))
1447 tmp = numpy.sqrt(numpy.absolute(L))
1447 data_specwidth = (self.lambda_/(2*math.sqrt(2)*math.pi*self.ippSec*self.nCohInt))*tmp*numpy.sign(L)
1448 data_specwidth = (self.lambda_/(2*math.sqrt(2)*math.pi*self.ippSec*self.nCohInt))*tmp*numpy.sign(L)
1448 n = self.__profIndex
1449 n = self.__profIndex
1449
1450
1450 self.__buffer = numpy.zeros((self.__nch, self.__nProf,self.__nHeis), dtype='complex')
1451 self.__buffer = numpy.zeros((self.__nch, self.__nProf,self.__nHeis), dtype='complex')
1451 self.__profIndex = 0
1452 self.__profIndex = 0
1452 return data_power,data_intensity,data_velocity,data_snrPP,data_specwidth,n
1453 return data_power,data_intensity,data_velocity,data_snrPP,data_specwidth,n
1453
1454
1454
1455
1455 def pulsePairbyProfiles(self,dataOut):
1456 def pulsePairbyProfiles(self,dataOut):
1456
1457
1457 self.__dataReady = False
1458 self.__dataReady = False
1458 data_power = None
1459 data_power = None
1459 data_intensity = None
1460 data_intensity = None
1460 data_velocity = None
1461 data_velocity = None
1461 data_specwidth = None
1462 data_specwidth = None
1462 data_snrPP = None
1463 data_snrPP = None
1463 self.putData(data=dataOut.data)
1464 self.putData(data=dataOut.data)
1464 if self.__profIndex == self.n:
1465 if self.__profIndex == self.n:
1465 data_power,data_intensity, data_velocity,data_snrPP,data_specwidth, n = self.pushData(dataOut=dataOut)
1466 data_power,data_intensity, data_velocity,data_snrPP,data_specwidth, n = self.pushData(dataOut=dataOut)
1466 self.__dataReady = True
1467 self.__dataReady = True
1467
1468
1468 return data_power, data_intensity, data_velocity, data_snrPP, data_specwidth
1469 return data_power, data_intensity, data_velocity, data_snrPP, data_specwidth
1469
1470
1470
1471
1471 def pulsePairOp(self, dataOut, datatime= None):
1472 def pulsePairOp(self, dataOut, datatime= None):
1472
1473
1473 if self.__initime == None:
1474 if self.__initime == None:
1474 self.__initime = datatime
1475 self.__initime = datatime
1475 data_power, data_intensity, data_velocity, data_snrPP, data_specwidth = self.pulsePairbyProfiles(dataOut)
1476 data_power, data_intensity, data_velocity, data_snrPP, data_specwidth = self.pulsePairbyProfiles(dataOut)
1476 self.__lastdatatime = datatime
1477 self.__lastdatatime = datatime
1477
1478
1478 if data_power is None:
1479 if data_power is None:
1479 return None, None, None,None,None,None
1480 return None, None, None,None,None,None
1480
1481
1481 avgdatatime = self.__initime
1482 avgdatatime = self.__initime
1482 deltatime = datatime - self.__lastdatatime
1483 deltatime = datatime - self.__lastdatatime
1483 self.__initime = datatime
1484 self.__initime = datatime
1484
1485
1485 return data_power, data_intensity, data_velocity, data_snrPP, data_specwidth, avgdatatime
1486 return data_power, data_intensity, data_velocity, data_snrPP, data_specwidth, avgdatatime
1486
1487
1487 def run(self, dataOut,n = None,removeDC= False, overlapping= False,**kwargs):
1488 def run(self, dataOut,n = None,removeDC= False, overlapping= False,**kwargs):
1488
1489
1489 if not self.isConfig:
1490 if not self.isConfig:
1490 self.setup(dataOut = dataOut, n = n , removeDC=removeDC , **kwargs)
1491 self.setup(dataOut = dataOut, n = n , removeDC=removeDC , **kwargs)
1491 self.isConfig = True
1492 self.isConfig = True
1492 data_power, data_intensity, data_velocity,data_snrPP,data_specwidth, avgdatatime = self.pulsePairOp(dataOut, dataOut.utctime)
1493 data_power, data_intensity, data_velocity,data_snrPP,data_specwidth, avgdatatime = self.pulsePairOp(dataOut, dataOut.utctime)
1493 dataOut.flagNoData = True
1494 dataOut.flagNoData = True
1494
1495
1495 if self.__dataReady:
1496 if self.__dataReady:
1496 dataOut.nCohInt *= self.n
1497 dataOut.nCohInt *= self.n
1497 dataOut.dataPP_POW = data_intensity # S
1498 dataOut.dataPP_POW = data_intensity # S
1498 dataOut.dataPP_POWER = data_power # P
1499 dataOut.dataPP_POWER = data_power # P
1499 dataOut.dataPP_DOP = data_velocity
1500 dataOut.dataPP_DOP = data_velocity
1500 dataOut.dataPP_SNR = data_snrPP
1501 dataOut.dataPP_SNR = data_snrPP
1501 dataOut.dataPP_WIDTH = data_specwidth
1502 dataOut.dataPP_WIDTH = data_specwidth
1502 dataOut.PRFbyAngle = self.n #numero de PRF*cada angulo rotado que equivale a un tiempo.
1503 dataOut.PRFbyAngle = self.n #numero de PRF*cada angulo rotado que equivale a un tiempo.
1503 dataOut.utctime = avgdatatime
1504 dataOut.utctime = avgdatatime
1504 dataOut.flagNoData = False
1505 dataOut.flagNoData = False
1505 return dataOut
1506 return dataOut
1506
1507
1507
1508
1508
1509
1509 # import collections
1510 # import collections
1510 # from scipy.stats import mode
1511 # from scipy.stats import mode
1511 #
1512 #
1512 # class Synchronize(Operation):
1513 # class Synchronize(Operation):
1513 #
1514 #
1514 # isConfig = False
1515 # isConfig = False
1515 # __profIndex = 0
1516 # __profIndex = 0
1516 #
1517 #
1517 # def __init__(self, **kwargs):
1518 # def __init__(self, **kwargs):
1518 #
1519 #
1519 # Operation.__init__(self, **kwargs)
1520 # Operation.__init__(self, **kwargs)
1520 # # self.isConfig = False
1521 # # self.isConfig = False
1521 # self.__powBuffer = None
1522 # self.__powBuffer = None
1522 # self.__startIndex = 0
1523 # self.__startIndex = 0
1523 # self.__pulseFound = False
1524 # self.__pulseFound = False
1524 #
1525 #
1525 # def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
1526 # def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
1526 #
1527 #
1527 # #Read data
1528 # #Read data
1528 #
1529 #
1529 # powerdB = dataOut.getPower(channel = channel)
1530 # powerdB = dataOut.getPower(channel = channel)
1530 # noisedB = dataOut.getNoise(channel = channel)[0]
1531 # noisedB = dataOut.getNoise(channel = channel)[0]
1531 #
1532 #
1532 # self.__powBuffer.extend(powerdB.flatten())
1533 # self.__powBuffer.extend(powerdB.flatten())
1533 #
1534 #
1534 # dataArray = numpy.array(self.__powBuffer)
1535 # dataArray = numpy.array(self.__powBuffer)
1535 #
1536 #
1536 # filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
1537 # filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
1537 #
1538 #
1538 # maxValue = numpy.nanmax(filteredPower)
1539 # maxValue = numpy.nanmax(filteredPower)
1539 #
1540 #
1540 # if maxValue < noisedB + 10:
1541 # if maxValue < noisedB + 10:
1541 # #No se encuentra ningun pulso de transmision
1542 # #No se encuentra ningun pulso de transmision
1542 # return None
1543 # return None
1543 #
1544 #
1544 # maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
1545 # maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
1545 #
1546 #
1546 # if len(maxValuesIndex) < 2:
1547 # if len(maxValuesIndex) < 2:
1547 # #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
1548 # #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
1548 # return None
1549 # return None
1549 #
1550 #
1550 # phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
1551 # phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
1551 #
1552 #
1552 # #Seleccionar solo valores con un espaciamiento de nSamples
1553 # #Seleccionar solo valores con un espaciamiento de nSamples
1553 # pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
1554 # pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
1554 #
1555 #
1555 # if len(pulseIndex) < 2:
1556 # if len(pulseIndex) < 2:
1556 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1557 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1557 # return None
1558 # return None
1558 #
1559 #
1559 # spacing = pulseIndex[1:] - pulseIndex[:-1]
1560 # spacing = pulseIndex[1:] - pulseIndex[:-1]
1560 #
1561 #
1561 # #remover senales que se distancien menos de 10 unidades o muestras
1562 # #remover senales que se distancien menos de 10 unidades o muestras
1562 # #(No deberian existir IPP menor a 10 unidades)
1563 # #(No deberian existir IPP menor a 10 unidades)
1563 #
1564 #
1564 # realIndex = numpy.where(spacing > 10 )[0]
1565 # realIndex = numpy.where(spacing > 10 )[0]
1565 #
1566 #
1566 # if len(realIndex) < 2:
1567 # if len(realIndex) < 2:
1567 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1568 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1568 # return None
1569 # return None
1569 #
1570 #
1570 # #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
1571 # #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
1571 # realPulseIndex = pulseIndex[realIndex]
1572 # realPulseIndex = pulseIndex[realIndex]
1572 #
1573 #
1573 # period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
1574 # period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
1574 #
1575 #
1575 # print "IPP = %d samples" %period
1576 # print "IPP = %d samples" %period
1576 #
1577 #
1577 # self.__newNSamples = dataOut.nHeights #int(period)
1578 # self.__newNSamples = dataOut.nHeights #int(period)
1578 # self.__startIndex = int(realPulseIndex[0])
1579 # self.__startIndex = int(realPulseIndex[0])
1579 #
1580 #
1580 # return 1
1581 # return 1
1581 #
1582 #
1582 #
1583 #
1583 # def setup(self, nSamples, nChannels, buffer_size = 4):
1584 # def setup(self, nSamples, nChannels, buffer_size = 4):
1584 #
1585 #
1585 # self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
1586 # self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
1586 # maxlen = buffer_size*nSamples)
1587 # maxlen = buffer_size*nSamples)
1587 #
1588 #
1588 # bufferList = []
1589 # bufferList = []
1589 #
1590 #
1590 # for i in range(nChannels):
1591 # for i in range(nChannels):
1591 # bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) + numpy.NAN,
1592 # bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) + numpy.NAN,
1592 # maxlen = buffer_size*nSamples)
1593 # maxlen = buffer_size*nSamples)
1593 #
1594 #
1594 # bufferList.append(bufferByChannel)
1595 # bufferList.append(bufferByChannel)
1595 #
1596 #
1596 # self.__nSamples = nSamples
1597 # self.__nSamples = nSamples
1597 # self.__nChannels = nChannels
1598 # self.__nChannels = nChannels
1598 # self.__bufferList = bufferList
1599 # self.__bufferList = bufferList
1599 #
1600 #
1600 # def run(self, dataOut, channel = 0):
1601 # def run(self, dataOut, channel = 0):
1601 #
1602 #
1602 # if not self.isConfig:
1603 # if not self.isConfig:
1603 # nSamples = dataOut.nHeights
1604 # nSamples = dataOut.nHeights
1604 # nChannels = dataOut.nChannels
1605 # nChannels = dataOut.nChannels
1605 # self.setup(nSamples, nChannels)
1606 # self.setup(nSamples, nChannels)
1606 # self.isConfig = True
1607 # self.isConfig = True
1607 #
1608 #
1608 # #Append new data to internal buffer
1609 # #Append new data to internal buffer
1609 # for thisChannel in range(self.__nChannels):
1610 # for thisChannel in range(self.__nChannels):
1610 # bufferByChannel = self.__bufferList[thisChannel]
1611 # bufferByChannel = self.__bufferList[thisChannel]
1611 # bufferByChannel.extend(dataOut.data[thisChannel])
1612 # bufferByChannel.extend(dataOut.data[thisChannel])
1612 #
1613 #
1613 # if self.__pulseFound:
1614 # if self.__pulseFound:
1614 # self.__startIndex -= self.__nSamples
1615 # self.__startIndex -= self.__nSamples
1615 #
1616 #
1616 # #Finding Tx Pulse
1617 # #Finding Tx Pulse
1617 # if not self.__pulseFound:
1618 # if not self.__pulseFound:
1618 # indexFound = self.__findTxPulse(dataOut, channel)
1619 # indexFound = self.__findTxPulse(dataOut, channel)
1619 #
1620 #
1620 # if indexFound == None:
1621 # if indexFound == None:
1621 # dataOut.flagNoData = True
1622 # dataOut.flagNoData = True
1622 # return
1623 # return
1623 #
1624 #
1624 # self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
1625 # self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
1625 # self.__pulseFound = True
1626 # self.__pulseFound = True
1626 # self.__startIndex = indexFound
1627 # self.__startIndex = indexFound
1627 #
1628 #
1628 # #If pulse was found ...
1629 # #If pulse was found ...
1629 # for thisChannel in range(self.__nChannels):
1630 # for thisChannel in range(self.__nChannels):
1630 # bufferByChannel = self.__bufferList[thisChannel]
1631 # bufferByChannel = self.__bufferList[thisChannel]
1631 # #print self.__startIndex
1632 # #print self.__startIndex
1632 # x = numpy.array(bufferByChannel)
1633 # x = numpy.array(bufferByChannel)
1633 # self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
1634 # self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
1634 #
1635 #
1635 # deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1636 # deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1636 # dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
1637 # dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
1637 # # dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
1638 # # dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
1638 #
1639 #
1639 # dataOut.data = self.__arrayBuffer
1640 # dataOut.data = self.__arrayBuffer
1640 #
1641 #
1641 # self.__startIndex += self.__newNSamples
1642 # self.__startIndex += self.__newNSamples
1642 #
1643 #
1643 # return
1644 # return
1644 class SSheightProfiles(Operation):
1645 class SSheightProfiles(Operation):
1645
1646
1646 step = None
1647 step = None
1647 nsamples = None
1648 nsamples = None
1648 bufferShape = None
1649 bufferShape = None
1649 profileShape = None
1650 profileShape = None
1650 sshProfiles = None
1651 sshProfiles = None
1651 profileIndex = None
1652 profileIndex = None
1652
1653
1653 def __init__(self, **kwargs):
1654 def __init__(self, **kwargs):
1654
1655
1655 Operation.__init__(self, **kwargs)
1656 Operation.__init__(self, **kwargs)
1656 self.isConfig = False
1657 self.isConfig = False
1657
1658
1658 def setup(self,dataOut ,step = None , nsamples = None):
1659 def setup(self,dataOut ,step = None , nsamples = None):
1659
1660
1660 if step == None and nsamples == None:
1661 if step == None and nsamples == None:
1661 raise ValueError("step or nheights should be specified ...")
1662 raise ValueError("step or nheights should be specified ...")
1662
1663
1663 self.step = step
1664 self.step = step
1664 self.nsamples = nsamples
1665 self.nsamples = nsamples
1665 self.__nChannels = dataOut.nChannels
1666 self.__nChannels = dataOut.nChannels
1666 self.__nProfiles = dataOut.nProfiles
1667 self.__nProfiles = dataOut.nProfiles
1667 self.__nHeis = dataOut.nHeights
1668 self.__nHeis = dataOut.nHeights
1668 shape = dataOut.data.shape #nchannels, nprofiles, nsamples
1669 shape = dataOut.data.shape #nchannels, nprofiles, nsamples
1669
1670
1670 residue = (shape[1] - self.nsamples) % self.step
1671 residue = (shape[1] - self.nsamples) % self.step
1671 if residue != 0:
1672 if residue != 0:
1672 print("The residue is %d, step=%d should be multiple of %d to avoid loss of %d samples"%(residue,step,shape[1] - self.nsamples,residue))
1673 print("The residue is %d, step=%d should be multiple of %d to avoid loss of %d samples"%(residue,step,shape[1] - self.nsamples,residue))
1673
1674
1674 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1675 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1675 numberProfile = self.nsamples
1676 numberProfile = self.nsamples
1676 numberSamples = (shape[1] - self.nsamples)/self.step
1677 numberSamples = (shape[1] - self.nsamples)/self.step
1677
1678
1678 self.bufferShape = int(shape[0]), int(numberSamples), int(numberProfile) # nchannels, nsamples , nprofiles
1679 self.bufferShape = int(shape[0]), int(numberSamples), int(numberProfile) # nchannels, nsamples , nprofiles
1679 self.profileShape = int(shape[0]), int(numberProfile), int(numberSamples) # nchannels, nprofiles, nsamples
1680 self.profileShape = int(shape[0]), int(numberProfile), int(numberSamples) # nchannels, nprofiles, nsamples
1680
1681
1681 self.buffer = numpy.zeros(self.bufferShape , dtype=numpy.complex)
1682 self.buffer = numpy.zeros(self.bufferShape , dtype=numpy.complex)
1682 self.sshProfiles = numpy.zeros(self.profileShape, dtype=numpy.complex)
1683 self.sshProfiles = numpy.zeros(self.profileShape, dtype=numpy.complex)
1683
1684
1684 def run(self, dataOut, step, nsamples, code = None, repeat = None):
1685 def run(self, dataOut, step, nsamples, code = None, repeat = None):
1685 dataOut.flagNoData = True
1686 dataOut.flagNoData = True
1686
1687
1687 profileIndex = None
1688 profileIndex = None
1688 #print("nProfiles, nHeights ",dataOut.nProfiles, dataOut.nHeights)
1689 #print("nProfiles, nHeights ",dataOut.nProfiles, dataOut.nHeights)
1689 #print(dataOut.getFreqRange(1)/1000.)
1690 #print(dataOut.getFreqRange(1)/1000.)
1690 #exit(1)
1691 #exit(1)
1691 if dataOut.flagDataAsBlock:
1692 if dataOut.flagDataAsBlock:
1692 dataOut.data = numpy.average(dataOut.data,axis=1)
1693 dataOut.data = numpy.average(dataOut.data,axis=1)
1693 #print("jee")
1694 #print("jee")
1694 dataOut.flagDataAsBlock = False
1695 dataOut.flagDataAsBlock = False
1695 if not self.isConfig:
1696 if not self.isConfig:
1696 self.setup(dataOut, step=step , nsamples=nsamples)
1697 self.setup(dataOut, step=step , nsamples=nsamples)
1697 #print("Setup done")
1698 #print("Setup done")
1698 self.isConfig = True
1699 self.isConfig = True
1699
1700
1700
1701
1701 if code is not None:
1702 if code is not None:
1702 code = numpy.array(code)
1703 code = numpy.array(code)
1703 code_block = code
1704 code_block = code
1704
1705
1705 if repeat is not None:
1706 if repeat is not None:
1706 code_block = numpy.repeat(code_block, repeats=repeat, axis=1)
1707 code_block = numpy.repeat(code_block, repeats=repeat, axis=1)
1707 #print(code_block.shape)
1708 #print(code_block.shape)
1708 for i in range(self.buffer.shape[1]):
1709 for i in range(self.buffer.shape[1]):
1709
1710
1710 if code is not None:
1711 if code is not None:
1711 self.buffer[:,i] = dataOut.data[:,i*self.step:i*self.step + self.nsamples]*code_block
1712 self.buffer[:,i] = dataOut.data[:,i*self.step:i*self.step + self.nsamples]*code_block
1712
1713
1713 else:
1714 else:
1714
1715
1715 self.buffer[:,i] = dataOut.data[:,i*self.step:i*self.step + self.nsamples]#*code[dataOut.profileIndex,:]
1716 self.buffer[:,i] = dataOut.data[:,i*self.step:i*self.step + self.nsamples]#*code[dataOut.profileIndex,:]
1716
1717
1717 #self.buffer[:,j,self.__nHeis-j*self.step - self.nheights:self.__nHeis-j*self.step] = numpy.flip(dataOut.data[:,j*self.step:j*self.step + self.nheights])
1718 #self.buffer[:,j,self.__nHeis-j*self.step - self.nheights:self.__nHeis-j*self.step] = numpy.flip(dataOut.data[:,j*self.step:j*self.step + self.nheights])
1718
1719
1719 for j in range(self.buffer.shape[0]):
1720 for j in range(self.buffer.shape[0]):
1720 self.sshProfiles[j] = numpy.transpose(self.buffer[j])
1721 self.sshProfiles[j] = numpy.transpose(self.buffer[j])
1721
1722
1722 profileIndex = self.nsamples
1723 profileIndex = self.nsamples
1723 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1724 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1724 ippSeconds = (deltaHeight*1.0e-6)/(0.15)
1725 ippSeconds = (deltaHeight*1.0e-6)/(0.15)
1725 #print("ippSeconds, dH: ",ippSeconds,deltaHeight)
1726 #print("ippSeconds, dH: ",ippSeconds,deltaHeight)
1726 try:
1727 try:
1727 if dataOut.concat_m is not None:
1728 if dataOut.concat_m is not None:
1728 ippSeconds= ippSeconds/float(dataOut.concat_m)
1729 ippSeconds= ippSeconds/float(dataOut.concat_m)
1729 #print "Profile concat %d"%dataOut.concat_m
1730 #print "Profile concat %d"%dataOut.concat_m
1730 except:
1731 except:
1731 pass
1732 pass
1732
1733
1733 dataOut.data = self.sshProfiles
1734 dataOut.data = self.sshProfiles
1734 dataOut.flagNoData = False
1735 dataOut.flagNoData = False
1735 dataOut.heightList = numpy.arange(self.buffer.shape[1]) *self.step*deltaHeight + dataOut.heightList[0]
1736 dataOut.heightList = numpy.arange(self.buffer.shape[1]) *self.step*deltaHeight + dataOut.heightList[0]
1736 dataOut.nProfiles = int(dataOut.nProfiles*self.nsamples)
1737 dataOut.nProfiles = int(dataOut.nProfiles*self.nsamples)
1737
1738
1738 dataOut.profileIndex = profileIndex
1739 dataOut.profileIndex = profileIndex
1739 dataOut.flagDataAsBlock = True
1740 dataOut.flagDataAsBlock = True
1740 dataOut.ippSeconds = ippSeconds
1741 dataOut.ippSeconds = ippSeconds
1741 dataOut.step = self.step
1742 dataOut.step = self.step
1742 #print(numpy.shape(dataOut.data))
1743 #print(numpy.shape(dataOut.data))
1743 #exit(1)
1744 #exit(1)
1744
1745
1745 return dataOut
1746 return dataOut
1746 ################################################################################3############################3
1747 ################################################################################3############################3
1747 ################################################################################3############################3
1748 ################################################################################3############################3
1748 ################################################################################3############################3
1749 ################################################################################3############################3
1749 ################################################################################3############################3
1750 ################################################################################3############################3
1750
1751
1751 class SSheightProfiles2(Operation):
1752 class SSheightProfiles2(Operation):
1752 '''
1753 '''
1753 Procesa por perfiles y por bloques
1754 Procesa por perfiles y por bloques
1754 '''
1755 '''
1755
1756
1756 step = None
1757 step = None
1757 nsamples = None
1758 nsamples = None
1758 bufferShape = None
1759 bufferShape = None
1759 profileShape = None
1760 profileShape = None
1760 sshProfiles = None
1761 sshProfiles = None
1761 profileIndex = None
1762 profileIndex = None
1763 deltaHeight = None
1764 init_range = None
1762
1765
1763 def __init__(self, **kwargs):
1766 def __init__(self, **kwargs):
1764
1767
1765 Operation.__init__(self, **kwargs)
1768 Operation.__init__(self, **kwargs)
1766 self.isConfig = False
1769 self.isConfig = False
1767
1770
1768 def setup(self,dataOut ,step = None , nsamples = None):
1771 def setup(self,dataOut ,step = None , nsamples = None):
1769
1772
1770 if step == None and nsamples == None:
1773 if step == None and nsamples == None:
1771 raise ValueError("step or nheights should be specified ...")
1774 raise ValueError("step or nheights should be specified ...")
1772
1775
1773 self.step = step
1776 self.step = step
1774 self.nsamples = nsamples
1777 self.nsamples = nsamples
1775 self.__nChannels = int(dataOut.nChannels)
1778 self.__nChannels = int(dataOut.nChannels)
1776 self.__nProfiles = int(dataOut.nProfiles)
1779 self.__nProfiles = int(dataOut.nProfiles)
1777 self.__nHeis = int(dataOut.nHeights)
1780 self.__nHeis = int(dataOut.nHeights)
1778
1781
1779 residue = (self.__nHeis - self.nsamples) % self.step
1782 residue = (self.__nHeis - self.nsamples) % self.step
1780 if residue != 0:
1783 if residue != 0:
1781 print("The residue is %d, step=%d should be multiple of %d to avoid loss of %d samples"%(residue,step,shape[1] - self.nsamples,residue))
1784 print("The residue is %d, step=%d should be multiple of %d to avoid loss of %d samples"%(residue,step,shape[1] - self.nsamples,residue))
1782
1785
1783 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1786 self.deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1787 self.init_range = dataOut.heightList[0]
1784 #numberProfile = self.nsamples
1788 #numberProfile = self.nsamples
1785 self.numberSamples = (self.__nHeis - self.nsamples)/self.step
1789 self.numberSamples = (self.__nHeis - self.nsamples)/self.step
1786
1790
1787 self.new_nHeights = self.numberSamples
1791 self.new_nHeights = self.numberSamples
1788
1792
1789 self.bufferShape = int(self.__nChannels), int(self.numberSamples), int(self.nsamples) # nchannels, nsamples , nprofiles
1793 self.bufferShape = int(self.__nChannels), int(self.numberSamples), int(self.nsamples) # nchannels, nsamples , nprofiles
1790 self.profileShape = int(self.__nChannels), int(self.nsamples), int(self.numberSamples) # nchannels, nprofiles, nsamples
1794 self.profileShape = int(self.__nChannels), int(self.nsamples), int(self.numberSamples) # nchannels, nprofiles, nsamples
1791
1795
1792 self.buffer = numpy.zeros(self.bufferShape , dtype=numpy.complex)
1796 self.buffer = numpy.zeros(self.bufferShape , dtype=numpy.complex)
1793 self.sshProfiles = numpy.zeros(self.profileShape, dtype=numpy.complex)
1797 self.sshProfiles = numpy.zeros(self.profileShape, dtype=numpy.complex)
1794
1798
1795 def getNewProfiles(self, data, code=None, repeat=None):
1799 def getNewProfiles(self, data, code=None, repeat=None):
1796
1800
1797 if code is not None:
1801 if code is not None:
1798 code = numpy.array(code)
1802 code = numpy.array(code)
1799 code_block = code
1803 code_block = code
1800
1804
1801 if repeat is not None:
1805 if repeat is not None:
1802 code_block = numpy.repeat(code_block, repeats=repeat, axis=1)
1806 code_block = numpy.repeat(code_block, repeats=repeat, axis=1)
1803 #print("buff, data, :",self.buffer.shape, data.shape)
1807 if data.ndim == 2:
1808 data = data.reshape(1,1,self.__nHeis )
1809 #print("buff, data, :",self.buffer.shape, data.shape,self.sshProfiles.shape)
1804 for i in range(int(self.new_nHeights)): #nuevas alturas
1810 for i in range(int(self.new_nHeights)): #nuevas alturas
1805 if code is not None:
1811 if code is not None:
1806 self.buffer[:,i] = data[:,i*self.step:i*self.step + self.nsamples]*code_block
1812 self.buffer[:,i,:] = data[:,:,i*self.step:i*self.step + self.nsamples]*code_block
1807 else:
1813 else:
1808 self.buffer[:,i] = data[:,i*self.step:i*self.step + self.nsamples]#*code[dataOut.profileIndex,:]
1814 self.buffer[:,i,:] = data[:,:,i*self.step:i*self.step + self.nsamples]#*code[dataOut.profileIndex,:]
1809
1815
1810 for j in range(self.__nChannels): #en los cananles
1816 for j in range(self.__nChannels): #en los cananles
1811 self.sshProfiles[j] = numpy.transpose(self.buffer[j])
1817 self.sshProfiles[j,:,:] = numpy.transpose(self.buffer[j,:,:])
1812
1818 #print("new profs Done")
1813
1819
1814
1820
1815
1821
1816 def run(self, dataOut, step, nsamples, code = None, repeat = None):
1822 def run(self, dataOut, step, nsamples, code = None, repeat = None):
1817
1823
1824 if dataOut.flagNoData == True:
1825 return dataOut
1818 dataOut.flagNoData = True
1826 dataOut.flagNoData = True
1819 #print("init data shape:", dataOut.data.shape)
1827 #print("init data shape:", dataOut.data.shape)
1820 #print("ch: {} prof: {} hs: {}".format(int(dataOut.nChannels),
1828 #print("ch: {} prof: {} hs: {}".format(int(dataOut.nChannels),
1821 # int(dataOut.nProfiles),int(dataOut.nHeights)))
1829 # int(dataOut.nProfiles),int(dataOut.nHeights)))
1822
1830
1823 profileIndex = None
1831 profileIndex = None
1824 # if not dataOut.flagDataAsBlock:
1832 # if not dataOut.flagDataAsBlock:
1825 # dataOut.nProfiles = 1
1833 # dataOut.nProfiles = 1
1826
1834
1827 if not self.isConfig:
1835 if not self.isConfig:
1828 self.setup(dataOut, step=step , nsamples=nsamples)
1836 self.setup(dataOut, step=step , nsamples=nsamples)
1829 #print("Setup done")
1837 #print("Setup done")
1830 self.isConfig = True
1838 self.isConfig = True
1831
1839
1832 dataBlock = None
1840 dataBlock = None
1833
1841
1834 nprof = 1
1842 nprof = 1
1835 if dataOut.flagDataAsBlock:
1843 if dataOut.flagDataAsBlock:
1836 nprof = int(dataOut.nProfiles)
1844 nprof = int(dataOut.nProfiles)
1837
1845
1838 #print("dataOut nProfiles:", dataOut.nProfiles)
1846 #print("dataOut nProfiles:", dataOut.nProfiles)
1839 for profile in range(nprof):
1847 for profile in range(nprof):
1840 if dataOut.flagDataAsBlock:
1848 if dataOut.flagDataAsBlock:
1849 #print("read blocks")
1841 self.getNewProfiles(dataOut.data[:,profile,:], code=code, repeat=repeat)
1850 self.getNewProfiles(dataOut.data[:,profile,:], code=code, repeat=repeat)
1842 else:
1851 else:
1843 self.getNewProfiles(dataOut.data, code=code, repeat=repeat)
1852 #print("read profiles")
1853 self.getNewProfiles(dataOut.data, code=code, repeat=repeat) #only one channe
1844 if profile == 0:
1854 if profile == 0:
1845 dataBlock = self.sshProfiles.copy()
1855 dataBlock = self.sshProfiles.copy()
1846 else: #by blocks
1856 else: #by blocks
1847 dataBlock = numpy.concatenate((dataBlock,self.sshProfiles), axis=1) #profile axis
1857 dataBlock = numpy.concatenate((dataBlock,self.sshProfiles), axis=1) #profile axis
1848 print("by blocks: ",dataBlock.shape, self.sshProfiles.shape)
1858 #print("by blocks: ",dataBlock.shape, self.sshProfiles.shape)
1849
1859
1850 profileIndex = self.nsamples
1860 profileIndex = self.nsamples
1851 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1861 #deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1852 ippSeconds = (deltaHeight*1.0e-6)/(0.15)
1862 ippSeconds = (self.deltaHeight*1.0e-6)/(0.15)
1853
1863
1854
1864
1855 dataOut.data = dataBlock
1865 dataOut.data = dataBlock
1856 dataOut.heightList = numpy.arange(self.new_nHeights) *self.step*deltaHeight + dataOut.heightList[0]
1866 #print("show me: ",self.step,self.deltaHeight, dataOut.heightList, self.new_nHeights)
1857 #print("show me: ",dataOut.nHeights, self.new_nHeights)
1867 dataOut.heightList = numpy.arange(int(self.new_nHeights)) *self.step*self.deltaHeight + self.init_range
1858 #dataOut.nHeights = int(self.new_nHeights)
1868
1859 dataOut.ippSeconds = ippSeconds
1869 dataOut.ippSeconds = ippSeconds
1860 dataOut.step = self.step
1870 dataOut.step = self.step
1861 dataOut.flagNoData = False
1871 dataOut.flagNoData = False
1862 if dataOut.flagDataAsBlock:
1872 if dataOut.flagDataAsBlock:
1863 dataOut.nProfiles = int(dataOut.nProfiles*self.nsamples)
1873 dataOut.nProfiles = int(dataOut.nProfiles*self.nsamples)
1864
1874
1865 else:
1875 else:
1866 dataOut.nProfiles = int(self.nsamples)
1876 dataOut.nProfiles = int(self.nsamples)
1867 dataOut.profileIndex = dataOut.nProfiles
1877 dataOut.profileIndex = dataOut.nProfiles
1868 dataOut.flagDataAsBlock = True
1878 dataOut.flagDataAsBlock = True
1869
1879
1870 dataBlock = None
1880 dataBlock = None
1871 #print("new data shape:", dataOut.data.shape)
1881 #print("new data shape:", dataOut.data.shape)
1872
1882
1873 return dataOut
1883 return dataOut
1874
1884
1875
1885
1876
1886
1877
1887
1878 #import skimage.color
1888 #import skimage.color
1879 #import skimage.io
1889 #import skimage.io
1880 import matplotlib.pyplot as plt
1890 #import matplotlib.pyplot as plt
1881
1891
1882 class clean2DArray(Operation):
1892 class removeProfileByFaradayHS(Operation):
1883 '''
1893 '''
1884
1894
1885 '''
1895 '''
1886 isConfig = False
1896 isConfig = False
1887 n = None
1897 n = None
1888 __timeInterval = None
1898
1889 __profIndex = 0
1899 __profIndex = 0
1890 __byTime = False
1900
1891 __dataReady = False
1901 __dataReady = False
1892 __buffer_data = []
1902 __buffer_data = []
1893 __buffer_times = []
1903 __buffer_times = []
1894 __initime = None
1904 __initime = None
1895 __count_exec = 0
1905 __count_exec = 0
1896 __profIndex = 0
1906 __profIndex = 0
1897 buffer = None
1907 buffer = None
1898 lenProfileOut = 1
1908 lenProfileOut = 1
1899
1909
1900 init_prof = 0
1910 init_prof = 0
1901 end_prof = 0
1911 end_prof = 0
1902 n_profiles = 0
1912 n_profiles = 0
1903 first_utcBlock = None
1913 first_utcBlock = None
1914 outliers_IDs_list = []
1904 __dh = 0
1915 __dh = 0
1905
1916
1906 def __init__(self, **kwargs):
1917 def __init__(self, **kwargs):
1907
1918
1908 Operation.__init__(self, **kwargs)
1919 Operation.__init__(self, **kwargs)
1909 self.isConfig = False
1920 self.isConfig = False
1910
1921
1911 def setup(self,dataOut, n=None , timeInterval=None):
1922 def setup(self,dataOut, n=None , navg=0.8, profileMargin=50,thHistOutlier=3, minHei=None, maxHei=None):
1912
1923
1913 if n == None and timeInterval == None:
1924 if n == None and timeInterval == None:
1914 raise ValueError("nprofiles or timeInterval should be specified ...")
1925 raise ValueError("nprofiles or timeInterval should be specified ...")
1915
1926
1916 if n != None:
1927 if n != None:
1917 self.n = n
1928 self.n = n
1918 self.__byTime = False
1919 else:
1920 self.__timeInterval = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
1921 self.n = 9999
1922 self.__byTime = True
1923
1929
1930 self.navg = navg
1931 self.profileMargin = profileMargin
1932 self.thHistOutlier = thHistOutlier
1924 self.__profIndex = 0
1933 self.__profIndex = 0
1925 self.buffer = None
1934 self.buffer = None
1926 self.lenProfileOut = 1
1935 self.lenProfileOut = 1
1927
1936 self.n_prof_released = 0
1937 self.heightList = dataOut.heightList
1928 self.init_prof = 0
1938 self.init_prof = 0
1929 self.end_prof = 0
1939 self.end_prof = 0
1930 self.n_profiles = 0
1940 self.n_profiles = 0
1931 self.first_utcBlock = None
1941 self.first_utcBlock = None
1932 self.__dh = dataOut.heightList[1] - dataOut.heightList[0]
1942 self.__dh = dataOut.heightList[1] - dataOut.heightList[0]
1943 minHei = minHei
1944 maxHei = maxHei
1945 if minHei==None :
1946 minHei = dataOut.heightList[0]
1947 if maxHei==None :
1948 maxHei = dataOut.heightList[-1]
1949 self.minHei_idx,self.maxHei_idx = getHei_index(minHei, maxHei, dataOut.heightList)
1950
1951
1952 def filterSatsProfiles(self):
1953 data = self.__buffer_data
1954 #print(data.shape)
1955 nChannels, profiles, heights = data.shape
1956 indexes=[]
1957 outliers_IDs=[]
1958 for c in range(nChannels):
1959 for h in range(self.minHei_idx, self.maxHei_idx):
1960 power = data[c,:,h] * numpy.conjugate(data[c,:,h])
1961 power = power.real
1962 #power = (numpy.abs(data[c,:,h].real))
1963 sortdata = numpy.sort(power, axis=None)
1964 sortID=power.argsort()
1965 index = _noise.hildebrand_sekhon2(sortdata,self.navg) #0.75-> buen valor
1966
1967 indexes.append(index)
1968 outliers_IDs=numpy.append(outliers_IDs,sortID[index:])
1969 # print(outliers_IDs)
1970 # fig,ax = plt.subplots()
1971 # #ax.set_title(str(k)+" "+str(j))
1972 # x=range(len(sortdata))
1973 # ax.scatter(x,sortdata)
1974 # ax.axvline(index)
1975 # plt.grid()
1976 # plt.show()
1977
1978 outliers_IDs = outliers_IDs.astype(numpy.dtype('int64'))
1979 outliers_IDs = numpy.unique(outliers_IDs)
1980 outs_lines = numpy.sort(outliers_IDs)
1981 # #print("outliers Ids: ", outs_lines, outs_lines.shape)
1982 #hist, bin_edges = numpy.histogram(outs_lines, bins=10, density=True)
1983
1984
1985 #Agrupando el histograma de outliers,
1986 my_bins = numpy.linspace(0,9600, 96, endpoint=False)
1987
1988
1989 hist, bins = numpy.histogram(outs_lines,bins=my_bins)
1990 hist_outliers_indexes = numpy.where(hist > self.thHistOutlier) #es outlier
1991 #print(hist_outliers_indexes[0])
1992 bins_outliers_indexes = [int(i) for i in bins[hist_outliers_indexes]] #
1993 #print(bins_outliers_indexes)
1994 outlier_loc_index = []
1995
1996 #outlier_loc_index = [k for k in range(bins_outliers_indexes[n]-50,bins_outliers_indexes[n+1]+50) for n in range(len(bins_outliers_indexes)-1) ]
1997 for n in range(len(bins_outliers_indexes)-1):
1998 #outlier_loc_index = [k for k in range(bins_outliers_indexes[n]-50,bins_outliers_indexes[n+1]+50)]
1999 for k in range(bins_outliers_indexes[n]-self.profileMargin,bins_outliers_indexes[n+1]+self.profileMargin):
2000 outlier_loc_index.append(k)
2001
2002 outlier_loc_index = numpy.asarray(outlier_loc_index)
2003 #print(numpy.unique(outlier_loc_index))
2004
2005
2006
2007 # x, y = numpy.meshgrid(numpy.arange(profiles), self.heightList)
2008 # fig, ax = plt.subplots(1,2,figsize=(8, 6))
2009 #
2010 # dat = data[0,:,:].real
2011 # m = numpy.nanmean(dat)
2012 # o = numpy.nanstd(dat)
2013 # #print(m, o, x.shape, y.shape)
2014 # c = ax[0].pcolormesh(x, y, dat.T, cmap ='YlGnBu', vmin = (m-2*o), vmax = (m+2*o))
2015 # ax[0].vlines(outs_lines,200,600, linestyles='dashed', label = 'outs', color='w')
2016 # fig.colorbar(c)
2017 # ax[0].vlines(outlier_loc_index,650,750, linestyles='dashed', label = 'outs', color='r')
2018 # ax[1].hist(outs_lines,bins=my_bins)
2019 # plt.show()
2020
2021
2022 self.outliers_IDs_list = numpy.unique(outlier_loc_index)
2023 return data
1933
2024
1934 def cleanOutliersByBlock(self):
2025 def cleanOutliersByBlock(self):
1935 #print(self.__buffer_data[0].shape)
2026 #print(self.__buffer_data[0].shape)
1936 data = self.__buffer_data#.copy()
2027 data = self.__buffer_data#.copy()
1937 #print("cleaning shape inpt: ",data.shape)
2028 #print("cleaning shape inpt: ",data.shape)
1938
2029 '''
1939 self.__buffer_data = []
2030 self.__buffer_data = []
1940
2031
1941 spectrum = numpy.fft.fft2(data, axes=(0,2))
2032 spectrum = numpy.fft.fft2(data, axes=(0,2))
1942 #print("spc : ",spectrum.shape)
2033 #print("spc : ",spectrum.shape)
1943 (nch,nsamples, nh) = spectrum.shape
2034 (nch,nsamples, nh) = spectrum.shape
1944
2035 data2 = None
1945 #nch =
2036 #print(data.shape)
1946
2037 spectrum2 = spectrum.copy()
1947 for ch in range(nch):
2038 for ch in range(nch):
1948 dh = self.__dh
2039 dh = self.__dh
1949 dt1 = (dh*1.0e-6)/(0.15)
2040 dt1 = (dh*1.0e-6)/(0.15)
1950 dt2 = self.__buffer_times[1]-self.__buffer_times[0]
2041 dt2 = self.__buffer_times[1]-self.__buffer_times[0]
1951 freqv = numpy.fft.fftfreq(nh, d=dt1)
2042 freqv = numpy.fft.fftfreq(nh, d=dt1)
1952 freqh = numpy.fft.fftfreq(self.n, d=dt2)
2043 freqh = numpy.fft.fftfreq(self.n, d=dt2)
1953 #print("spc loop: ")
2044 #print("spc loop: ")
2045
2046
2047
1954 x, y = numpy.meshgrid(numpy.sort(freqh),numpy.sort(freqv))
2048 x, y = numpy.meshgrid(numpy.sort(freqh),numpy.sort(freqv))
1955 z = numpy.abs(spectrum[ch,:,:])
2049 z = numpy.abs(spectrum[ch,:,:])
1956
2050 # Find all peaks higher than the 98th percentile
2051 peaks = z < numpy.percentile(z, 98)
2052 #print(peaks)
2053 # Set those peak coefficients to zero
2054 spectrum2 = spectrum2 * peaks.astype(int)
2055 data2 = numpy.fft.ifft2(spectrum2)
1957
2056
1958 dat = numpy.log10(z.T)
2057 dat = numpy.log10(z.T)
1959 m = numpy.mean(dat)
2058 dat2 = numpy.log10(spectrum2.T)
1960 o = numpy.std(dat)
1961 fig, ax = plt.subplots(figsize=(8, 6))
1962
1963 c = ax.pcolormesh(x, y, dat, cmap ='YlGnBu', vmin = (m-2*o), vmax = (m+2*o))
1964 #c = ax.pcolor( z.T , cmap ='gray', vmin = (m-2*o), vmax = (m+2*o))
1965 date_time = datetime.datetime.fromtimestamp(self.__buffer_times[0]).strftime('%Y-%m-%d %H:%M:%S.%f')
1966 #strftime('%Y-%m-%d %H:%M:%S')
1967 ax.set_title('Spectrum magnitude '+date_time)
1968 fig.canvas.set_window_title('Spectrum magnitude {} '.format(self.n)+date_time)
1969 fig.colorbar(c)
1970
2059
1971 plt.show()
2060 # m = numpy.mean(dat)
2061 # o = numpy.std(dat)
2062 # fig, ax = plt.subplots(2,1,figsize=(8, 6))
2063 #
2064 # c = ax[0].pcolormesh(x, y, dat, cmap ='YlGnBu', vmin = (m-2*o), vmax = (m+2*o))
2065 # #c = ax.pcolor( z.T , cmap ='gray', vmin = (m-2*o), vmax = (m+2*o))
2066 # date_time = datetime.datetime.fromtimestamp(self.__buffer_times[0]).strftime('%Y-%m-%d %H:%M:%S.%f')
2067 # #strftime('%Y-%m-%d %H:%M:%S')
2068 # ax[0].set_title('Spectrum magnitude '+date_time)
2069 # fig.canvas.set_window_title('Spectrum magnitude {} '.format(self.n)+date_time)
2070 #
2071 #
2072 # c = ax[1].pcolormesh(x, y, dat, cmap ='YlGnBu', vmin = (m-2*o), vmax = (m+2*o))
2073 # fig.colorbar(c)
2074 # plt.show()
1972
2075
2076 #print(data2.shape)
1973
2077
2078 data = data2
1974
2079
1975 #cleanBlock = numpy.fft.ifft2(spectrum, axes=(0,2)).reshape()
2080 #cleanBlock = numpy.fft.ifft2(spectrum, axes=(0,2)).reshape()
2081 '''
2082 #print("cleanOutliersByBlock Done")
1976
2083
1977 print("cleanOutliersByBlock Done")
2084 return self.filterSatsProfiles()
1978 return data
1979
1980 def byTime(self, data, datatime):
1981
1982 self.__dataReady = False
1983
1984 self.fillBuffer(data, datatime)
1985 dataBlock = None
1986
1987 if (datatime - self.__initime) >= self.__timeInterval:
1988 dataBlock = self.cleanOutliersByBlock()
1989 self.__dataReady = True
1990 self.n = self.__profIndex
1991 return dataBlock
1992
2085
1993 def byProfiles(self, data, datatime):
1994 self.__dataReady = False
1995
2086
1996 self.fillBuffer(data, datatime)
1997 dataBlock = None
1998
1999 if self.__profIndex == self.n:
2000 #print("apnd : ",data)
2001 dataBlock = self.cleanOutliersByBlock()
2002 self.__dataReady = True
2003
2004 return dataBlock
2005
2087
2006 def fillBuffer(self, data, datatime):
2088 def fillBuffer(self, data, datatime):
2007
2089
2008 if self.__profIndex == 0:
2090 if self.__profIndex == 0:
2009 self.__buffer_data = data.copy()
2091 self.__buffer_data = data.copy()
2010
2092
2011 else:
2093 else:
2012 self.__buffer_data = numpy.concatenate((self.__buffer_data,data), axis=1)#en perfiles
2094 self.__buffer_data = numpy.concatenate((self.__buffer_data,data), axis=1)#en perfiles
2013 self.__profIndex += 1
2095 self.__profIndex += 1
2014 self.__buffer_times.append(datatime)
2096 #self.__buffer_times.append(datatime)
2015
2097
2016 def getData(self, data, datatime=None):
2098 def getData(self, data, datatime=None):
2017
2099
2018 if self.__profIndex == 0:
2100 if self.__profIndex == 0:
2019 self.__initime = datatime
2101 self.__initime = datatime
2020
2102
2021 if self.__byTime:
2022 dataBlock = self.byTime(data, datatime)
2023 else:
2024 dataBlock = self.byProfiles(data, datatime)
2025
2103
2104 self.__dataReady = False
2105
2106 self.fillBuffer(data, datatime)
2107 dataBlock = None
2108
2109 if self.__profIndex == self.n:
2110 #print("apnd : ",data)
2111 #dataBlock = self.cleanOutliersByBlock()
2112 dataBlock = self.filterSatsProfiles()
2113 self.__dataReady = True
2026
2114
2115 return dataBlock
2027
2116
2028 if dataBlock is None:
2117 if dataBlock is None:
2029 return None
2118 return None, None
2030
2119
2031
2120
2032
2121
2033 return dataBlock
2122 return dataBlock
2034
2123
2035 def releaseBlock(self, dataOut):
2124 def releaseBlock(self):
2036
2125
2037 if self.n % self.lenProfileOut != 0:
2126 if self.n % self.lenProfileOut != 0:
2038 raise ValueError("lenProfileOut %d must be submultiple of nProfiles %d" %(self.lenProfileOut, self.n_profiles))
2127 raise ValueError("lenProfileOut %d must be submultiple of nProfiles %d" %(self.lenProfileOut, self.n_profiles))
2039 return None
2128 return None
2040
2129
2041 dataOut.data = self.buffer[:,self.init_prof:self.end_prof:,:] #ch, prof, alt
2130 data = self.buffer[:,self.init_prof:self.end_prof:,:] #ch, prof, alt
2042 dataOut.profileIndex = self.lenProfileOut
2131
2043 dataOut.utctime = self.first_utcBlock + self.init_prof*dataOut.ippSeconds
2044 self.init_prof = self.end_prof
2132 self.init_prof = self.end_prof
2045 self.end_prof += self.lenProfileOut
2133 self.end_prof += self.lenProfileOut
2046 print("data release shape: ",dataOut.data.shape, self.end_prof)
2134 #print("data release shape: ",dataOut.data.shape, self.end_prof)
2135 self.n_prof_released += 1
2047
2136
2048 if self.end_prof >= (self.n +self.lenProfileOut):
2049 self.init_prof = 0
2050 self.__profIndex = 0
2051 self.buffer = None
2052 dataOut.buffer_empty = True
2053 return dataOut
2054
2137
2055 def run(self, dataOut, n=None, timeInterval=None, nProfilesOut=1):
2138 #print("f_no_data ", dataOut.flagNoData)
2139 return data
2140
2141 def run(self, dataOut, n=None, navg=0.8, nProfilesOut=1, profile_margin=50,th_hist_outlier=3,minHei=None, maxHei=None):
2056 #print("run op buffer 2D")
2142 #print("run op buffer 2D")
2057 self.nChannels = dataOut.nChannels
2143 self.nChannels = dataOut.nChannels
2058 self.nHeights = dataOut.nHeights
2144 self.nHeights = dataOut.nHeights
2059
2145
2060 if not self.isConfig:
2146 if not self.isConfig:
2061 self.setup(dataOut,n=n, timeInterval=timeInterval)
2147 #print("init p idx: ", dataOut.profileIndex )
2148 self.setup(dataOut,n=n, navg=navg,profileMargin=profile_margin,
2149 thHistOutlier=th_hist_outlier,minHei=minHei, maxHei=maxHei)
2062 self.isConfig = True
2150 self.isConfig = True
2063
2151
2064 dataBlock = None
2152 dataBlock = None
2065
2153
2066 if not dataOut.buffer_empty: #hay datos acumulados
2154 if not dataOut.buffer_empty: #hay datos acumulados
2067
2155
2068 if self.init_prof == 0:
2156 if self.init_prof == 0:
2157 self.n_prof_released = 0
2069 self.lenProfileOut = nProfilesOut
2158 self.lenProfileOut = nProfilesOut
2070 dataOut.flagNoData = False
2159 dataOut.flagNoData = False
2071 #print("tp 2 ",dataOut.data.shape)
2160 #print("tp 2 ",dataOut.data.shape)
2072 #(ch, self.n_profiles, nh) = self.buffer.shape
2161 #(ch, self.n_profiles, nh) = self.buffer.shape
2073 #print("tp 3 ",self.dataOut.data.shape)
2162 #print("tp 3 ",self.dataOut.data.shape)
2074 #print("rel: ",self.buffer[:,-1,:])
2163 #print("rel: ",self.buffer[:,-1,:])
2075 self.init_prof = 0
2164 self.init_prof = 0
2076 self.end_prof = self.lenProfileOut
2165 self.end_prof = self.lenProfileOut
2077 self.first_utcBlock = dataOut.utctime
2166
2078 dataOut.nProfiles = self.lenProfileOut
2167 dataOut.nProfiles = self.lenProfileOut
2079 dataOut.error = False
2168 if nProfilesOut == 1:
2169 dataOut.flagDataAsBlock = False
2170 else:
2080 dataOut.flagDataAsBlock = True
2171 dataOut.flagDataAsBlock = True
2081 #print("prof: ",self.init_prof)
2172 #print("prof: ",self.init_prof)
2082 dataOut.flagNoData = False
2173 dataOut.flagNoData = False
2083 return self.releaseBlock(dataOut)
2174 if numpy.isin(self.n_prof_released, self.outliers_IDs_list):
2175 #print("omitting: ", self.n_prof_released)
2176 dataOut.flagNoData = True
2177
2178 dataOut.utctime = self.first_utcBlock + self.init_prof*dataOut.ippSeconds
2179 #dataOut.data = self.releaseBlock()
2180 #########################################################3
2181 if self.n % self.lenProfileOut != 0:
2182 raise ValueError("lenProfileOut %d must be submultiple of nProfiles %d" %(self.lenProfileOut, self.n_profiles))
2183 return None
2184
2185 dataOut.data = self.buffer[:,self.init_prof:self.end_prof:,:] #ch, prof, alt
2186
2187 self.init_prof = self.end_prof
2188 self.end_prof += self.lenProfileOut
2189 #print("data release shape: ",dataOut.data.shape, self.end_prof)
2190 self.n_prof_released += 1
2191
2192 if self.end_prof >= (self.n +self.lenProfileOut):
2193
2194 self.init_prof = 0
2195 self.__profIndex = 0
2196 self.buffer = None
2197 dataOut.buffer_empty = True
2198 self.outliers_IDs_list = []
2199 self.n_prof_released = 0
2200 dataOut.flagNoData = False #enviar ultimo aunque sea outlier :(
2201 #print("cleaning...", dataOut.buffer_empty)
2202 dataOut.profileIndex = 0 #self.lenProfileOut
2203 ####################################################################
2204 return dataOut
2084
2205
2085
2206
2086 #print("tp 223 ",dataOut.data.shape)
2207 #print("tp 223 ",dataOut.data.shape)
2087 dataOut.flagNoData = True
2208 dataOut.flagNoData = True
2088
2209
2089
2210
2090
2211
2091 try:
2212 try:
2092 #dataBlock = self.getData(dataOut.data.reshape(self.nChannels,1,self.nHeights), dataOut.utctime)
2213 #dataBlock = self.getData(dataOut.data.reshape(self.nChannels,1,self.nHeights), dataOut.utctime)
2093 dataBlock = self.getData(numpy.reshape(dataOut.data,(self.nChannels,1,self.nHeights)), dataOut.utctime)
2214 dataBlock = self.getData(numpy.reshape(dataOut.data,(self.nChannels,1,self.nHeights)), dataOut.utctime)
2094 self.__count_exec +=1
2215 self.__count_exec +=1
2095 except Exception as e:
2216 except Exception as e:
2096 print("Error getting profiles data",self.__count_exec )
2217 print("Error getting profiles data",self.__count_exec )
2097 print(e)
2218 print(e)
2098 sys.exit()
2219 sys.exit()
2099
2220
2100 if self.__dataReady:
2221 if self.__dataReady:
2222 #print("omitting: ", len(self.outliers_IDs_list))
2101 self.__count_exec = 0
2223 self.__count_exec = 0
2102 #dataOut.data =
2224 #dataOut.data =
2103 self.buffer = numpy.flip(dataBlock, axis=1)
2225 #self.buffer = numpy.flip(dataBlock, axis=1)
2104
2226 self.buffer = dataBlock
2227 self.first_utcBlock = self.__initime
2105 dataOut.utctime = self.__initime
2228 dataOut.utctime = self.__initime
2106 dataOut.nProfiles = self.__profIndex
2229 dataOut.nProfiles = self.__profIndex
2107 #dataOut.flagNoData = False
2230 #dataOut.flagNoData = False
2231 self.init_prof = 0
2108 self.__profIndex = 0
2232 self.__profIndex = 0
2109 self.__initime = None
2233 self.__initime = None
2110 dataBlock = None
2234 dataBlock = None
2111 self.__buffer_times = []
2235 self.__buffer_times = []
2112 dataOut.error = False
2236 dataOut.error = False
2113 dataOut.useInputBuffer = True
2237 dataOut.useInputBuffer = True
2114 dataOut.buffer_empty = False
2238 dataOut.buffer_empty = False
2115 print("1 ch: {} prof: {} hs: {}".format(int(dataOut.nChannels),
2239 #print("1 ch: {} prof: {} hs: {}".format(int(dataOut.nChannels),int(dataOut.nProfiles),int(dataOut.nHeights)))
2116 int(dataOut.nProfiles),int(dataOut.nHeights)))
2240
2117 #return None
2118
2241
2119
2242
2120 #print(self.__count_exec)
2243 #print(self.__count_exec)
2121
2244
2122 return dataOut
2245 return dataOut
2123
2246
2124 class CleanProfileSats(Operation):
2247 class RemoveProfileSats(Operation):
2125 '''
2248 '''
2126 Omite los perfiles contaminados con señal de satelites,
2249 Omite los perfiles contaminados con señal de satelites,
2127 In: minHei = min_sat_range
2250 In: minHei = min_sat_range
2128 max_sat_range
2251 max_sat_range
2129 min_hei_ref
2252 min_hei_ref
2130 max_hei_ref
2253 max_hei_ref
2131 th = diference between profiles mean, ref and sats
2254 th = diference between profiles mean, ref and sats
2132 Out:
2255 Out:
2133 profile clean
2256 profile clean
2134 '''
2257 '''
2135
2258
2136 isConfig = False
2259 isConfig = False
2137 min_sats = 0
2260 min_sats = 0
2138 max_sats = 999999999
2261 max_sats = 999999999
2139 min_ref= 0
2262 min_ref= 0
2140 max_ref= 9999999999
2263 max_ref= 9999999999
2141 needReshape = False
2264 needReshape = False
2142 count = 0
2265 count = 0
2143 thdB = 0
2266 thdB = 0
2144 byRanges = False
2267 byRanges = False
2145 min_sats = None
2268 min_sats = None
2146 max_sats = None
2269 max_sats = None
2270 noise = 0
2147
2271
2148 def __init__(self, **kwargs):
2272 def __init__(self, **kwargs):
2149
2273
2150 Operation.__init__(self, **kwargs)
2274 Operation.__init__(self, **kwargs)
2151 self.isConfig = False
2275 self.isConfig = False
2152
2276
2153
2277
2154 def setup(self, dataOut, minHei, maxHei, minRef, maxRef, th, thdB, rangeHeiList):
2278 def setup(self, dataOut, minHei, maxHei, minRef, maxRef, th, thdB, rangeHeiList):
2155
2279
2156 if rangeHeiList!=None:
2280 if rangeHeiList!=None:
2157 self.byRanges = True
2281 self.byRanges = True
2158 else:
2282 else:
2159 if minHei==None or maxHei==None :
2283 if minHei==None or maxHei==None :
2160 raise ValueError("Parameters heights are required")
2284 raise ValueError("Parameters heights are required")
2161 if minRef==None or maxRef==None:
2285 if minRef==None or maxRef==None:
2162 raise ValueError("Parameters heights are required")
2286 raise ValueError("Parameters heights are required")
2163
2287
2164 if self.byRanges:
2288 if self.byRanges:
2165 self.min_sats = []
2289 self.min_sats = []
2166 self.max_sats = []
2290 self.max_sats = []
2167 for min,max in rangeHeiList:
2291 for min,max in rangeHeiList:
2168 a,b = getHei_index(min, max, dataOut.heightList)
2292 a,b = getHei_index(min, max, dataOut.heightList)
2169 self.min_sats.append(a)
2293 self.min_sats.append(a)
2170 self.max_sats.append(b)
2294 self.max_sats.append(b)
2171 else:
2295 else:
2172 self.min_sats, self.max_sats = getHei_index(minHei, maxHei, dataOut.heightList)
2296 self.min_sats, self.max_sats = getHei_index(minHei, maxHei, dataOut.heightList)
2173 self.min_ref, self.max_ref = getHei_index(minRef, maxRef, dataOut.heightList)
2297 self.min_ref, self.max_ref = getHei_index(minRef, maxRef, dataOut.heightList)
2174 self.th = th
2298 self.th = th
2175 self.thdB = thdB
2299 self.thdB = thdB
2176 self.isConfig = True
2300 self.isConfig = True
2177
2301
2178
2302
2179 def compareRanges(self,data, minHei,maxHei):
2303 def compareRanges(self,data, minHei,maxHei):
2180 ref = data[0,self.min_ref:self.max_ref] * numpy.conjugate(data[0,self.min_ref:self.max_ref])
2304
2181 p_ref = 10*numpy.log10(ref.real)
2305 # ref = data[0,self.min_ref:self.max_ref] * numpy.conjugate(data[0,self.min_ref:self.max_ref])
2182 m_ref = numpy.mean(p_ref)
2306 # p_ref = 10*numpy.log10(ref.real)
2307 # m_ref = numpy.mean(p_ref)
2308
2309 m_ref = self.noise
2183
2310
2184 sats = data[0,minHei:maxHei] * numpy.conjugate(data[0,minHei:maxHei])
2311 sats = data[0,minHei:maxHei] * numpy.conjugate(data[0,minHei:maxHei])
2185 p_sats = 10*numpy.log10(sats.real)
2312 p_sats = 10*numpy.log10(sats.real)
2186 m_sats = numpy.mean(p_sats)
2313 m_sats = numpy.mean(p_sats)
2187
2314
2188 if m_sats > (m_ref + self.th) and (m_sats > self.thdB):
2315 if m_sats > (m_ref + self.th): #and (m_sats > self.thdB):
2189 #print("msats: ",m_sats," mRef: ", m_ref, (m_sats - m_ref))
2316 #print("msats: ",m_sats," \tmRef: ", m_ref, "\t",(m_sats - m_ref))
2190 #print("Removing profiles...")
2317 #print("Removing profiles...")
2191 return False
2318 return False
2192
2319
2193 return True
2320 return True
2194
2321
2195 def isProfileClean(self, data):
2322 def isProfileClean(self, data):
2196 '''
2323 '''
2197 Analiza solo 1 canal, y descarta todos...
2324 Analiza solo 1 canal, y descarta todos...
2198 '''
2325 '''
2199
2326
2200 clean = True
2327 clean = True
2201
2328
2202 if self.byRanges:
2329 if self.byRanges:
2203
2330
2204 for n in range(len(self.min_sats)):
2331 for n in range(len(self.min_sats)):
2205 c = self.compareRanges(data,self.min_sats[n],self.max_sats[n])
2332 c = self.compareRanges(data,self.min_sats[n],self.max_sats[n])
2206 clean = clean and c
2333 clean = clean and c
2207 else:
2334 else:
2208
2335
2209 clean = (self.compareRanges(data, self.min_sats,self.max_sats))
2336 clean = (self.compareRanges(data, self.min_sats,self.max_sats))
2210
2337
2211 return clean
2338 return clean
2212
2339
2213
2340
2214
2341
2215 def run(self, dataOut, minHei=None, maxHei=None, minRef=None, maxRef=None, th=5, thdB=65, rangeHeiList=None):
2342 def run(self, dataOut, minHei=None, maxHei=None, minRef=None, maxRef=None, th=5, thdB=65, rangeHeiList=None):
2216 dataOut.flagNoData = True
2343 dataOut.flagNoData = True
2217
2344
2218 if not self.isConfig:
2345 if not self.isConfig:
2219 self.setup(dataOut, minHei, maxHei, minRef, maxRef, th, thdB, rangeHeiList)
2346 self.setup(dataOut, minHei, maxHei, minRef, maxRef, th, thdB, rangeHeiList)
2220 self.isConfig = True
2347 self.isConfig = True
2221
2348 #print(self.min_sats,self.max_sats)
2222 if dataOut.flagDataAsBlock:
2349 if dataOut.flagDataAsBlock:
2223 raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
2350 raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
2224
2351
2225 else:
2352 else:
2226
2353 self.noise =10*numpy.log10(dataOut.getNoisebyHildebrand(ymin_index=self.min_ref, ymax_index=self.max_ref))
2227 if not self.isProfileClean(dataOut.data):
2354 if not self.isProfileClean(dataOut.data):
2228 return dataOut
2355 return dataOut
2229 #dataOut.data = numpy.full((dataOut.nChannels,dataOut.nHeights),numpy.NAN)
2356 #dataOut.data = numpy.full((dataOut.nChannels,dataOut.nHeights),numpy.NAN)
2230 #self.count += 1
2357 #self.count += 1
2231
2358
2232 dataOut.flagNoData = False
2359 dataOut.flagNoData = False
2233
2360
2234 return dataOut
2361 return dataOut
General Comments 0
You need to be logged in to leave comments. Login now