##// END OF EJS Templates
schain-jc
RSS
Update version, fix kwargs for self operations (methods), Add SendToWeb...
Juan C. Espinoza -
r906:a52f011a763e
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,98 +1,102
1 # Byte-compiled / optimized / DLL files
1 # Byte-compiled / optimized / DLL files
2 __pycache__/
2 __pycache__/
3 *.py[cod]
3 *.py[cod]
4 *$py.class
4 *$py.class
5
5
6 # C extensions
6 # C extensions
7 *.so
7 *.so
8
8
9 # Distribution / packaging
9 # Distribution / packaging
10 .Python
10 .Python
11 env/
11 env/
12 build/
12 build/
13 develop-eggs/
13 develop-eggs/
14 dist/
14 dist/
15 downloads/
15 downloads/
16 eggs/
16 eggs/
17 .eggs/
17 .eggs/
18 lib/
18 lib/
19 lib64/
19 lib64/
20 parts/
20 parts/
21 sdist/
21 sdist/
22 var/
22 var/
23 wheels/
23 wheels/
24 *.egg-info/
24 *.egg-info/
25 .installed.cfg
25 .installed.cfg
26 *.egg
26 *.egg
27
27
28 # PyInstaller
28 # PyInstaller
29 # Usually these files are written by a python script from a template
29 # Usually these files are written by a python script from a template
30 # before PyInstaller builds the exe, so as to inject date/other infos into it.
30 # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 *.manifest
31 *.manifest
32 *.spec
32 *.spec
33
33
34 # Installer logs
34 # Installer logs
35 pip-log.txt
35 pip-log.txt
36 pip-delete-this-directory.txt
36 pip-delete-this-directory.txt
37
37
38 # Unit test / coverage reports
38 # Unit test / coverage reports
39 htmlcov/
39 htmlcov/
40 .tox/
40 .tox/
41 .coverage
41 .coverage
42 .coverage.*
42 .coverage.*
43 .cache
43 .cache
44 nosetests.xml
44 nosetests.xml
45 coverage.xml
45 coverage.xml
46 *,cover
46 *,cover
47 .hypothesis/
47 .hypothesis/
48
48
49 # Translations
49 # Translations
50 *.mo
50 *.mo
51 *.pot
51 *.pot
52
52
53 # Django stuff:
53 # Django stuff:
54 *.log
54 *.log
55 local_settings.py
55 local_settings.py
56
56
57 # Flask stuff:
57 # Flask stuff:
58 instance/
58 instance/
59 .webassets-cache
59 .webassets-cache
60
60
61 # Scrapy stuff:
61 # Scrapy stuff:
62 .scrapy
62 .scrapy
63
63
64 # Sphinx documentation
64 # Sphinx documentation
65 docs/_build/
65 docs/_build/
66
66
67 # PyBuilder
67 # PyBuilder
68 target/
68 target/
69
69
70 # Jupyter Notebook
70 # Jupyter Notebook
71 .ipynb_checkpoints
71 .ipynb_checkpoints
72
72
73 # pyenv
73 # pyenv
74 .python-version
74 .python-version
75
75
76 # celery beat schedule file
76 # celery beat schedule file
77 celerybeat-schedule
77 celerybeat-schedule
78
78
79 # SageMath parsed files
79 # SageMath parsed files
80 *.sage.py
80 *.sage.py
81
81
82 # dotenv
82 # dotenv
83 .env
83 .env
84
84
85 # virtualenv
85 # virtualenv
86 .venv
86 .venv
87 venv/
87 venv/
88 ENV/
88 ENV/
89
89
90 # Spyder project settings
90 # Spyder project settings
91 .spyderproject
91 .spyderproject
92 .spyproject
92 .spyproject
93
93
94 # Rope project settings
94 # Rope project settings
95 .ropeproject
95 .ropeproject
96
96
97 # mkdocs documentation
97 # mkdocs documentation
98 /site
98 /site
99
100 # eclipse
101 .project
102 .pydevproject
Show file before | Show file after | Hide comments
@@ -1,7 +1,7
1 '''
1 '''
2 Created on Feb 7, 2012
2 Created on Feb 7, 2012
3
3
4 @author $Author$
4 @author $Author$
5 @version $Id$
5 @version $Id$
6 '''
6 '''
7 __version__ = "2.2.5" No newline at end of file
7 __version__ = "2.3" No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1318 +1,1323
1 '''
1 '''
2 Created on September , 2012
2 Created on September , 2012
3 @author:
3 @author:
4 '''
4 '''
5
5
6 import sys
6 import sys
7 import ast
7 import ast
8 import datetime
8 import datetime
9 import traceback
9 import traceback
10 import math
10 import math
11 from multiprocessing import Process, Queue, cpu_count
11 from multiprocessing import Process, Queue, cpu_count
12
12
13 import schainpy
13 import schainpy
14 import schainpy.admin
14 import schainpy.admin
15
15
16 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
16 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
17 from xml.dom import minidom
17 from xml.dom import minidom
18
18
19 from schainpy.model import *
19 from schainpy.model import *
20 from time import sleep
20 from time import sleep
21
21
22 def prettify(elem):
22 def prettify(elem):
23 """Return a pretty-printed XML string for the Element.
23 """Return a pretty-printed XML string for the Element.
24 """
24 """
25 rough_string = tostring(elem, 'utf-8')
25 rough_string = tostring(elem, 'utf-8')
26 reparsed = minidom.parseString(rough_string)
26 reparsed = minidom.parseString(rough_string)
27 return reparsed.toprettyxml(indent=" ")
27 return reparsed.toprettyxml(indent=" ")
28
28
29 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, receiver=None):
29 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, receiver=None):
30 skip = 0
30 skip = 0
31 cursor = 0
31 cursor = 0
32 nFiles = None
32 nFiles = None
33 processes = []
33 processes = []
34
34
35 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
35 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
36 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
36 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
37 days = (dt2 - dt1).days
37 days = (dt2 - dt1).days
38 print days
38 print days
39 for day in range(days+1):
39 for day in range(days+1):
40 skip = 0
40 skip = 0
41 cursor = 0
41 cursor = 0
42 q = Queue()
42 q = Queue()
43 processes = []
43 processes = []
44 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
44 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
45 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
45 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
46 firstProcess.start()
46 firstProcess.start()
47 nFiles = q.get()
47 nFiles = q.get()
48 firstProcess.terminate()
48 firstProcess.terminate()
49 skip = int(math.ceil(nFiles/nProcess))
49 skip = int(math.ceil(nFiles/nProcess))
50 try:
50 try:
51 while True:
51 while True:
52 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
52 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
53 processes[cursor].start()
53 processes[cursor].start()
54 if nFiles < cursor*skip:
54 if nFiles < cursor*skip:
55 break
55 break
56 cursor += 1
56 cursor += 1
57 except KeyboardInterrupt:
57 except KeyboardInterrupt:
58 for process in processes:
58 for process in processes:
59 process.terminate()
59 process.terminate()
60 process.join()
60 process.join()
61 for process in processes:
61 for process in processes:
62 process.join()
62 process.join()
63 # process.terminate()
63 # process.terminate()
64 sleep(3)
64 sleep(3)
65
65
66 try:
66 try:
67 while True:
67 while True:
68 pass
68 pass
69 except KeyboardInterrupt:
69 except KeyboardInterrupt:
70 for process in processes:
70 for process in processes:
71 process.terminate()
71 process.terminate()
72 process.join()
72 process.join()
73
73
74 class ParameterConf():
74 class ParameterConf():
75
75
76 id = None
76 id = None
77 name = None
77 name = None
78 value = None
78 value = None
79 format = None
79 format = None
80
80
81 __formated_value = None
81 __formated_value = None
82
82
83 ELEMENTNAME = 'Parameter'
83 ELEMENTNAME = 'Parameter'
84
84
85 def __init__(self):
85 def __init__(self):
86
86
87 self.format = 'str'
87 self.format = 'str'
88
88
89 def getElementName(self):
89 def getElementName(self):
90
90
91 return self.ELEMENTNAME
91 return self.ELEMENTNAME
92
92
93 def getValue(self):
93 def getValue(self):
94
94
95 value = self.value
95 value = self.value
96 format = self.format
96 format = self.format
97
97
98 if self.__formated_value != None:
98 if self.__formated_value != None:
99
99
100 return self.__formated_value
100 return self.__formated_value
101
101
102 if format == 'obj':
102 if format == 'obj':
103 return value
103 return value
104
104
105 if format == 'str':
105 if format == 'str':
106 self.__formated_value = str(value)
106 self.__formated_value = str(value)
107 return self.__formated_value
107 return self.__formated_value
108
108
109 if value == '':
109 if value == '':
110 raise ValueError, "%s: This parameter value is empty" %self.name
110 raise ValueError, "%s: This parameter value is empty" %self.name
111
111
112 if format == 'list':
112 if format == 'list':
113 strList = value.split(',')
113 strList = value.split(',')
114
114
115 self.__formated_value = strList
115 self.__formated_value = strList
116
116
117 return self.__formated_value
117 return self.__formated_value
118
118
119 if format == 'intlist':
119 if format == 'intlist':
120 """
120 """
121 Example:
121 Example:
122 value = (0,1,2)
122 value = (0,1,2)
123 """
123 """
124
124
125 new_value = ast.literal_eval(value)
125 new_value = ast.literal_eval(value)
126
126
127 if type(new_value) not in (tuple, list):
127 if type(new_value) not in (tuple, list):
128 new_value = [int(new_value)]
128 new_value = [int(new_value)]
129
129
130 self.__formated_value = new_value
130 self.__formated_value = new_value
131
131
132 return self.__formated_value
132 return self.__formated_value
133
133
134 if format == 'floatlist':
134 if format == 'floatlist':
135 """
135 """
136 Example:
136 Example:
137 value = (0.5, 1.4, 2.7)
137 value = (0.5, 1.4, 2.7)
138 """
138 """
139
139
140 new_value = ast.literal_eval(value)
140 new_value = ast.literal_eval(value)
141
141
142 if type(new_value) not in (tuple, list):
142 if type(new_value) not in (tuple, list):
143 new_value = [float(new_value)]
143 new_value = [float(new_value)]
144
144
145 self.__formated_value = new_value
145 self.__formated_value = new_value
146
146
147 return self.__formated_value
147 return self.__formated_value
148
148
149 if format == 'date':
149 if format == 'date':
150 strList = value.split('/')
150 strList = value.split('/')
151 intList = [int(x) for x in strList]
151 intList = [int(x) for x in strList]
152 date = datetime.date(intList[0], intList[1], intList[2])
152 date = datetime.date(intList[0], intList[1], intList[2])
153
153
154 self.__formated_value = date
154 self.__formated_value = date
155
155
156 return self.__formated_value
156 return self.__formated_value
157
157
158 if format == 'time':
158 if format == 'time':
159 strList = value.split(':')
159 strList = value.split(':')
160 intList = [int(x) for x in strList]
160 intList = [int(x) for x in strList]
161 time = datetime.time(intList[0], intList[1], intList[2])
161 time = datetime.time(intList[0], intList[1], intList[2])
162
162
163 self.__formated_value = time
163 self.__formated_value = time
164
164
165 return self.__formated_value
165 return self.__formated_value
166
166
167 if format == 'pairslist':
167 if format == 'pairslist':
168 """
168 """
169 Example:
169 Example:
170 value = (0,1),(1,2)
170 value = (0,1),(1,2)
171 """
171 """
172
172
173 new_value = ast.literal_eval(value)
173 new_value = ast.literal_eval(value)
174
174
175 if type(new_value) not in (tuple, list):
175 if type(new_value) not in (tuple, list):
176 raise ValueError, "%s has to be a tuple or list of pairs" %value
176 raise ValueError, "%s has to be a tuple or list of pairs" %value
177
177
178 if type(new_value[0]) not in (tuple, list):
178 if type(new_value[0]) not in (tuple, list):
179 if len(new_value) != 2:
179 if len(new_value) != 2:
180 raise ValueError, "%s has to be a tuple or list of pairs" %value
180 raise ValueError, "%s has to be a tuple or list of pairs" %value
181 new_value = [new_value]
181 new_value = [new_value]
182
182
183 for thisPair in new_value:
183 for thisPair in new_value:
184 if len(thisPair) != 2:
184 if len(thisPair) != 2:
185 raise ValueError, "%s has to be a tuple or list of pairs" %value
185 raise ValueError, "%s has to be a tuple or list of pairs" %value
186
186
187 self.__formated_value = new_value
187 self.__formated_value = new_value
188
188
189 return self.__formated_value
189 return self.__formated_value
190
190
191 if format == 'multilist':
191 if format == 'multilist':
192 """
192 """
193 Example:
193 Example:
194 value = (0,1,2),(3,4,5)
194 value = (0,1,2),(3,4,5)
195 """
195 """
196 multiList = ast.literal_eval(value)
196 multiList = ast.literal_eval(value)
197
197
198 if type(multiList[0]) == int:
198 if type(multiList[0]) == int:
199 multiList = ast.literal_eval("(" + value + ")")
199 multiList = ast.literal_eval("(" + value + ")")
200
200
201 self.__formated_value = multiList
201 self.__formated_value = multiList
202
202
203 return self.__formated_value
203 return self.__formated_value
204
204
205 if format == 'bool':
205 if format == 'bool':
206 value = int(value)
206 value = int(value)
207
207
208 if format == 'int':
208 if format == 'int':
209 value = float(value)
209 value = float(value)
210
210
211 format_func = eval(format)
211 format_func = eval(format)
212
212
213 self.__formated_value = format_func(value)
213 self.__formated_value = format_func(value)
214
214
215 return self.__formated_value
215 return self.__formated_value
216
216
217 def updateId(self, new_id):
217 def updateId(self, new_id):
218
218
219 self.id = str(new_id)
219 self.id = str(new_id)
220
220
221 def setup(self, id, name, value, format='str'):
221 def setup(self, id, name, value, format='str'):
222
222
223 self.id = str(id)
223 self.id = str(id)
224 self.name = name
224 self.name = name
225 if format == 'obj':
225 if format == 'obj':
226 self.value = value
226 self.value = value
227 else:
227 else:
228 self.value = str(value)
228 self.value = str(value)
229 self.format = str.lower(format)
229 self.format = str.lower(format)
230
230
231 self.getValue()
231 self.getValue()
232
232
233 return 1
233 return 1
234
234
235 def update(self, name, value, format='str'):
235 def update(self, name, value, format='str'):
236
236
237 self.name = name
237 self.name = name
238 self.value = str(value)
238 self.value = str(value)
239 self.format = format
239 self.format = format
240
240
241 def makeXml(self, opElement):
241 def makeXml(self, opElement):
242 if self.name not in ('queue',):
242 if self.name not in ('queue',):
243 parmElement = SubElement(opElement, self.ELEMENTNAME)
243 parmElement = SubElement(opElement, self.ELEMENTNAME)
244 parmElement.set('id', str(self.id))
244 parmElement.set('id', str(self.id))
245 parmElement.set('name', self.name)
245 parmElement.set('name', self.name)
246 parmElement.set('value', self.value)
246 parmElement.set('value', self.value)
247 parmElement.set('format', self.format)
247 parmElement.set('format', self.format)
248
248
249 def readXml(self, parmElement):
249 def readXml(self, parmElement):
250
250
251 self.id = parmElement.get('id')
251 self.id = parmElement.get('id')
252 self.name = parmElement.get('name')
252 self.name = parmElement.get('name')
253 self.value = parmElement.get('value')
253 self.value = parmElement.get('value')
254 self.format = str.lower(parmElement.get('format'))
254 self.format = str.lower(parmElement.get('format'))
255
255
256 #Compatible with old signal chain version
256 #Compatible with old signal chain version
257 if self.format == 'int' and self.name == 'idfigure':
257 if self.format == 'int' and self.name == 'idfigure':
258 self.name = 'id'
258 self.name = 'id'
259
259
260 def printattr(self):
260 def printattr(self):
261
261
262 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
262 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
263
263
264 class OperationConf():
264 class OperationConf():
265
265
266 id = None
266 id = None
267 name = None
267 name = None
268 priority = None
268 priority = None
269 type = None
269 type = None
270
270
271 parmConfObjList = []
271 parmConfObjList = []
272
272
273 ELEMENTNAME = 'Operation'
273 ELEMENTNAME = 'Operation'
274
274
275 def __init__(self):
275 def __init__(self):
276
276
277 self.id = '0'
277 self.id = '0'
278 self.name = None
278 self.name = None
279 self.priority = None
279 self.priority = None
280 self.type = 'self'
280 self.type = 'self'
281
281
282
282
283 def __getNewId(self):
283 def __getNewId(self):
284
284
285 return int(self.id)*10 + len(self.parmConfObjList) + 1
285 return int(self.id)*10 + len(self.parmConfObjList) + 1
286
286
287 def updateId(self, new_id):
287 def updateId(self, new_id):
288
288
289 self.id = str(new_id)
289 self.id = str(new_id)
290
290
291 n = 1
291 n = 1
292 for parmObj in self.parmConfObjList:
292 for parmObj in self.parmConfObjList:
293
293
294 idParm = str(int(new_id)*10 + n)
294 idParm = str(int(new_id)*10 + n)
295 parmObj.updateId(idParm)
295 parmObj.updateId(idParm)
296
296
297 n += 1
297 n += 1
298
298
299 def getElementName(self):
299 def getElementName(self):
300
300
301 return self.ELEMENTNAME
301 return self.ELEMENTNAME
302
302
303 def getParameterObjList(self):
303 def getParameterObjList(self):
304
304
305 return self.parmConfObjList
305 return self.parmConfObjList
306
306
307 def getParameterObj(self, parameterName):
307 def getParameterObj(self, parameterName):
308
308
309 for parmConfObj in self.parmConfObjList:
309 for parmConfObj in self.parmConfObjList:
310
310
311 if parmConfObj.name != parameterName:
311 if parmConfObj.name != parameterName:
312 continue
312 continue
313
313
314 return parmConfObj
314 return parmConfObj
315
315
316 return None
316 return None
317
317
318 def getParameterObjfromValue(self, parameterValue):
318 def getParameterObjfromValue(self, parameterValue):
319
319
320 for parmConfObj in self.parmConfObjList:
320 for parmConfObj in self.parmConfObjList:
321
321
322 if parmConfObj.getValue() != parameterValue:
322 if parmConfObj.getValue() != parameterValue:
323 continue
323 continue
324
324
325 return parmConfObj.getValue()
325 return parmConfObj.getValue()
326
326
327 return None
327 return None
328
328
329 def getParameterValue(self, parameterName):
329 def getParameterValue(self, parameterName):
330
330
331 parameterObj = self.getParameterObj(parameterName)
331 parameterObj = self.getParameterObj(parameterName)
332
332
333 # if not parameterObj:
333 # if not parameterObj:
334 # return None
334 # return None
335
335
336 value = parameterObj.getValue()
336 value = parameterObj.getValue()
337
337
338 return value
338 return value
339
339
340
340
341 def getKwargs(self):
341 def getKwargs(self):
342
342
343 kwargs = {}
343 kwargs = {}
344
344
345 for parmConfObj in self.parmConfObjList:
345 for parmConfObj in self.parmConfObjList:
346 if self.name == 'run' and parmConfObj.name == 'datatype':
346 if self.name == 'run' and parmConfObj.name == 'datatype':
347 continue
347 continue
348
348
349 kwargs[parmConfObj.name] = parmConfObj.getValue()
349 kwargs[parmConfObj.name] = parmConfObj.getValue()
350
350
351 return kwargs
351 return kwargs
352
352
353 def setup(self, id, name, priority, type):
353 def setup(self, id, name, priority, type):
354
354
355 self.id = str(id)
355 self.id = str(id)
356 self.name = name
356 self.name = name
357 self.type = type
357 self.type = type
358 self.priority = priority
358 self.priority = priority
359
359
360 self.parmConfObjList = []
360 self.parmConfObjList = []
361
361
362 def removeParameters(self):
362 def removeParameters(self):
363
363
364 for obj in self.parmConfObjList:
364 for obj in self.parmConfObjList:
365 del obj
365 del obj
366
366
367 self.parmConfObjList = []
367 self.parmConfObjList = []
368
368
369 def addParameter(self, name, value, format='str'):
369 def addParameter(self, name, value, format='str'):
370
370
371 id = self.__getNewId()
371 id = self.__getNewId()
372
372
373 parmConfObj = ParameterConf()
373 parmConfObj = ParameterConf()
374 if not parmConfObj.setup(id, name, value, format):
374 if not parmConfObj.setup(id, name, value, format):
375 return None
375 return None
376
376
377 self.parmConfObjList.append(parmConfObj)
377 self.parmConfObjList.append(parmConfObj)
378
378
379 return parmConfObj
379 return parmConfObj
380
380
381 def changeParameter(self, name, value, format='str'):
381 def changeParameter(self, name, value, format='str'):
382
382
383 parmConfObj = self.getParameterObj(name)
383 parmConfObj = self.getParameterObj(name)
384 parmConfObj.update(name, value, format)
384 parmConfObj.update(name, value, format)
385
385
386 return parmConfObj
386 return parmConfObj
387
387
388 def makeXml(self, procUnitElement):
388 def makeXml(self, procUnitElement):
389
389
390 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
390 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
391 opElement.set('id', str(self.id))
391 opElement.set('id', str(self.id))
392 opElement.set('name', self.name)
392 opElement.set('name', self.name)
393 opElement.set('type', self.type)
393 opElement.set('type', self.type)
394 opElement.set('priority', str(self.priority))
394 opElement.set('priority', str(self.priority))
395
395
396 for parmConfObj in self.parmConfObjList:
396 for parmConfObj in self.parmConfObjList:
397 parmConfObj.makeXml(opElement)
397 parmConfObj.makeXml(opElement)
398
398
399 def readXml(self, opElement):
399 def readXml(self, opElement):
400
400
401 self.id = opElement.get('id')
401 self.id = opElement.get('id')
402 self.name = opElement.get('name')
402 self.name = opElement.get('name')
403 self.type = opElement.get('type')
403 self.type = opElement.get('type')
404 self.priority = opElement.get('priority')
404 self.priority = opElement.get('priority')
405
405
406 #Compatible with old signal chain version
406 #Compatible with old signal chain version
407 #Use of 'run' method instead 'init'
407 #Use of 'run' method instead 'init'
408 if self.type == 'self' and self.name == 'init':
408 if self.type == 'self' and self.name == 'init':
409 self.name = 'run'
409 self.name = 'run'
410
410
411 self.parmConfObjList = []
411 self.parmConfObjList = []
412
412
413 parmElementList = opElement.iter(ParameterConf().getElementName())
413 parmElementList = opElement.iter(ParameterConf().getElementName())
414
414
415 for parmElement in parmElementList:
415 for parmElement in parmElementList:
416 parmConfObj = ParameterConf()
416 parmConfObj = ParameterConf()
417 parmConfObj.readXml(parmElement)
417 parmConfObj.readXml(parmElement)
418
418
419 #Compatible with old signal chain version
419 #Compatible with old signal chain version
420 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
420 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
421 if self.type != 'self' and self.name == 'Plot':
421 if self.type != 'self' and self.name == 'Plot':
422 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
422 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
423 self.name = parmConfObj.value
423 self.name = parmConfObj.value
424 continue
424 continue
425
425
426 self.parmConfObjList.append(parmConfObj)
426 self.parmConfObjList.append(parmConfObj)
427
427
428 def printattr(self):
428 def printattr(self):
429
429
430 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
430 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
431 self.id,
431 self.id,
432 self.name,
432 self.name,
433 self.type,
433 self.type,
434 self.priority)
434 self.priority)
435
435
436 for parmConfObj in self.parmConfObjList:
436 for parmConfObj in self.parmConfObjList:
437 parmConfObj.printattr()
437 parmConfObj.printattr()
438
438
439 def createObject(self, plotter_queue=None):
439 def createObject(self, plotter_queue=None):
440
440
441 if self.type == 'self':
441
442 if self.type == 'self':
442 raise ValueError, "This operation type cannot be created"
443 raise ValueError, "This operation type cannot be created"
443
444
444 if self.type == 'plotter':
445 if self.type == 'plotter':
445 #Plotter(plotter_name)
446 #Plotter(plotter_name)
446 if not plotter_queue:
447 if not plotter_queue:
447 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
448 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
448
449
449 opObj = Plotter(self.name, plotter_queue)
450 opObj = Plotter(self.name, plotter_queue)
450
451
451 if self.type == 'external' or self.type == 'other':
452 if self.type == 'external' or self.type == 'other':
452 print self.name
453
453 className = eval(self.name)
454 className = eval(self.name)
454 kwargs = self.getKwargs()
455 kwargs = self.getKwargs()
455 print kwargs
456
456 opObj = className(**kwargs)
457 opObj = className(**kwargs)
457
458
458 return opObj
459 return opObj
459
460
460
461
461 class ProcUnitConf():
462 class ProcUnitConf():
462
463
463 id = None
464 id = None
464 name = None
465 name = None
465 datatype = None
466 datatype = None
466 inputId = None
467 inputId = None
467 parentId = None
468 parentId = None
468
469
469 opConfObjList = []
470 opConfObjList = []
470
471
471 procUnitObj = None
472 procUnitObj = None
472 opObjList = []
473 opObjList = []
473
474
474 ELEMENTNAME = 'ProcUnit'
475 ELEMENTNAME = 'ProcUnit'
475
476
476 def __init__(self):
477 def __init__(self):
477
478
478 self.id = None
479 self.id = None
479 self.datatype = None
480 self.datatype = None
480 self.name = None
481 self.name = None
481 self.inputId = None
482 self.inputId = None
482
483
483 self.opConfObjList = []
484 self.opConfObjList = []
484
485
485 self.procUnitObj = None
486 self.procUnitObj = None
486 self.opObjDict = {}
487 self.opObjDict = {}
487
488
488 def __getPriority(self):
489 def __getPriority(self):
489
490
490 return len(self.opConfObjList)+1
491 return len(self.opConfObjList)+1
491
492
492 def __getNewId(self):
493 def __getNewId(self):
493
494
494 return int(self.id)*10 + len(self.opConfObjList) + 1
495 return int(self.id)*10 + len(self.opConfObjList) + 1
495
496
496 def getElementName(self):
497 def getElementName(self):
497
498
498 return self.ELEMENTNAME
499 return self.ELEMENTNAME
499
500
500 def getId(self):
501 def getId(self):
501
502
502 return self.id
503 return self.id
503
504
504 def updateId(self, new_id, parentId=parentId):
505 def updateId(self, new_id, parentId=parentId):
505
506
506
507
507 new_id = int(parentId)*10 + (int(self.id) % 10)
508 new_id = int(parentId)*10 + (int(self.id) % 10)
508 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
509 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
509
510
510 #If this proc unit has not inputs
511 #If this proc unit has not inputs
511 if self.inputId == '0':
512 if self.inputId == '0':
512 new_inputId = 0
513 new_inputId = 0
513
514
514 n = 1
515 n = 1
515 for opConfObj in self.opConfObjList:
516 for opConfObj in self.opConfObjList:
516
517
517 idOp = str(int(new_id)*10 + n)
518 idOp = str(int(new_id)*10 + n)
518 opConfObj.updateId(idOp)
519 opConfObj.updateId(idOp)
519
520
520 n += 1
521 n += 1
521
522
522 self.parentId = str(parentId)
523 self.parentId = str(parentId)
523 self.id = str(new_id)
524 self.id = str(new_id)
524 self.inputId = str(new_inputId)
525 self.inputId = str(new_inputId)
525
526
526
527
527 def getInputId(self):
528 def getInputId(self):
528
529
529 return self.inputId
530 return self.inputId
530
531
531 def getOperationObjList(self):
532 def getOperationObjList(self):
532
533
533 return self.opConfObjList
534 return self.opConfObjList
534
535
535 def getOperationObj(self, name=None):
536 def getOperationObj(self, name=None):
536
537
537 for opConfObj in self.opConfObjList:
538 for opConfObj in self.opConfObjList:
538
539
539 if opConfObj.name != name:
540 if opConfObj.name != name:
540 continue
541 continue
541
542
542 return opConfObj
543 return opConfObj
543
544
544 return None
545 return None
545
546
546 def getOpObjfromParamValue(self, value=None):
547 def getOpObjfromParamValue(self, value=None):
547
548
548 for opConfObj in self.opConfObjList:
549 for opConfObj in self.opConfObjList:
549 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
550 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
550 continue
551 continue
551 return opConfObj
552 return opConfObj
552 return None
553 return None
553
554
554 def getProcUnitObj(self):
555 def getProcUnitObj(self):
555
556
556 return self.procUnitObj
557 return self.procUnitObj
557
558
558 def setup(self, id, name, datatype, inputId, parentId=None):
559 def setup(self, id, name, datatype, inputId, parentId=None):
559
560
560 #Compatible with old signal chain version
561 #Compatible with old signal chain version
561 if datatype==None and name==None:
562 if datatype==None and name==None:
562 raise ValueError, "datatype or name should be defined"
563 raise ValueError, "datatype or name should be defined"
563
564
564 if name==None:
565 if name==None:
565 if 'Proc' in datatype:
566 if 'Proc' in datatype:
566 name = datatype
567 name = datatype
567 else:
568 else:
568 name = '%sProc' %(datatype)
569 name = '%sProc' %(datatype)
569
570
570 if datatype==None:
571 if datatype==None:
571 datatype = name.replace('Proc','')
572 datatype = name.replace('Proc','')
572
573
573 self.id = str(id)
574 self.id = str(id)
574 self.name = name
575 self.name = name
575 self.datatype = datatype
576 self.datatype = datatype
576 self.inputId = inputId
577 self.inputId = inputId
577 self.parentId = parentId
578 self.parentId = parentId
578
579
579 self.opConfObjList = []
580 self.opConfObjList = []
580
581
581 self.addOperation(name='run', optype='self')
582 self.addOperation(name='run', optype='self')
582
583
583 def removeOperations(self):
584 def removeOperations(self):
584
585
585 for obj in self.opConfObjList:
586 for obj in self.opConfObjList:
586 del obj
587 del obj
587
588
588 self.opConfObjList = []
589 self.opConfObjList = []
589 self.addOperation(name='run')
590 self.addOperation(name='run')
590
591
591 def addParameter(self, **kwargs):
592 def addParameter(self, **kwargs):
592 '''
593 '''
593 Add parameters to "run" operation
594 Add parameters to "run" operation
594 '''
595 '''
595 opObj = self.opConfObjList[0]
596 opObj = self.opConfObjList[0]
596
597
597 opObj.addParameter(**kwargs)
598 opObj.addParameter(**kwargs)
598
599
599 return opObj
600 return opObj
600
601
601 def addOperation(self, name, optype='self'):
602 def addOperation(self, name, optype='self'):
602
603
603 id = self.__getNewId()
604 id = self.__getNewId()
604 priority = self.__getPriority()
605 priority = self.__getPriority()
605
606
606 opConfObj = OperationConf()
607 opConfObj = OperationConf()
607 opConfObj.setup(id, name=name, priority=priority, type=optype)
608 opConfObj.setup(id, name=name, priority=priority, type=optype)
608
609
609 self.opConfObjList.append(opConfObj)
610 self.opConfObjList.append(opConfObj)
610
611
611 return opConfObj
612 return opConfObj
612
613
613 def makeXml(self, projectElement):
614 def makeXml(self, projectElement):
614
615
615 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
616 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
616 procUnitElement.set('id', str(self.id))
617 procUnitElement.set('id', str(self.id))
617 procUnitElement.set('name', self.name)
618 procUnitElement.set('name', self.name)
618 procUnitElement.set('datatype', self.datatype)
619 procUnitElement.set('datatype', self.datatype)
619 procUnitElement.set('inputId', str(self.inputId))
620 procUnitElement.set('inputId', str(self.inputId))
620
621
621 for opConfObj in self.opConfObjList:
622 for opConfObj in self.opConfObjList:
622 opConfObj.makeXml(procUnitElement)
623 opConfObj.makeXml(procUnitElement)
623
624
624 def readXml(self, upElement):
625 def readXml(self, upElement):
625
626
626 self.id = upElement.get('id')
627 self.id = upElement.get('id')
627 self.name = upElement.get('name')
628 self.name = upElement.get('name')
628 self.datatype = upElement.get('datatype')
629 self.datatype = upElement.get('datatype')
629 self.inputId = upElement.get('inputId')
630 self.inputId = upElement.get('inputId')
630
631
631 if self.ELEMENTNAME == "ReadUnit":
632 if self.ELEMENTNAME == "ReadUnit":
632 self.datatype = self.datatype.replace("Reader", "")
633 self.datatype = self.datatype.replace("Reader", "")
633
634
634 if self.ELEMENTNAME == "ProcUnit":
635 if self.ELEMENTNAME == "ProcUnit":
635 self.datatype = self.datatype.replace("Proc", "")
636 self.datatype = self.datatype.replace("Proc", "")
636
637
637 if self.inputId == 'None':
638 if self.inputId == 'None':
638 self.inputId = '0'
639 self.inputId = '0'
639
640
640 self.opConfObjList = []
641 self.opConfObjList = []
641
642
642 opElementList = upElement.iter(OperationConf().getElementName())
643 opElementList = upElement.iter(OperationConf().getElementName())
643
644
644 for opElement in opElementList:
645 for opElement in opElementList:
645 opConfObj = OperationConf()
646 opConfObj = OperationConf()
646 opConfObj.readXml(opElement)
647 opConfObj.readXml(opElement)
647 self.opConfObjList.append(opConfObj)
648 self.opConfObjList.append(opConfObj)
648
649
649 def printattr(self):
650 def printattr(self):
650
651
651 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
652 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
652 self.id,
653 self.id,
653 self.name,
654 self.name,
654 self.datatype,
655 self.datatype,
655 self.inputId)
656 self.inputId)
656
657
657 for opConfObj in self.opConfObjList:
658 for opConfObj in self.opConfObjList:
658 opConfObj.printattr()
659 opConfObj.printattr()
659
660
660
661
661 def getKwargs(self):
662 def getKwargs(self):
662
663
663 opObj = self.opConfObjList[0]
664 opObj = self.opConfObjList[0]
664 kwargs = opObj.getKwargs()
665 kwargs = opObj.getKwargs()
665
666
666 return kwargs
667 return kwargs
667
668
668 def createObjects(self, plotter_queue=None):
669 def createObjects(self, plotter_queue=None):
669
670
670 className = eval(self.name)
671 className = eval(self.name)
671 kwargs = self.getKwargs()
672 kwargs = self.getKwargs()
672 procUnitObj = className(**kwargs)
673 procUnitObj = className(**kwargs)
673
674
674 for opConfObj in self.opConfObjList:
675 for opConfObj in self.opConfObjList:
675
676
676 if opConfObj.type == 'self':
677 if opConfObj.type=='self' and self.name=='run':
678 continue
679 elif opConfObj.type=='self':
680 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
677 continue
681 continue
678
682
679 opObj = opConfObj.createObject(plotter_queue)
683 opObj = opConfObj.createObject(plotter_queue)
680
684
681 self.opObjDict[opConfObj.id] = opObj
685 self.opObjDict[opConfObj.id] = opObj
686
682 procUnitObj.addOperation(opObj, opConfObj.id)
687 procUnitObj.addOperation(opObj, opConfObj.id)
683
688
684 self.procUnitObj = procUnitObj
689 self.procUnitObj = procUnitObj
685
690
686 return procUnitObj
691 return procUnitObj
687
692
688 def run(self):
693 def run(self):
689
694
690 is_ok = False
695 is_ok = False
691
696
692 for opConfObj in self.opConfObjList:
697 for opConfObj in self.opConfObjList:
693
698
694 kwargs = {}
699 kwargs = {}
695 for parmConfObj in opConfObj.getParameterObjList():
700 for parmConfObj in opConfObj.getParameterObjList():
696 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
701 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
697 continue
702 continue
698
703
699 kwargs[parmConfObj.name] = parmConfObj.getValue()
704 kwargs[parmConfObj.name] = parmConfObj.getValue()
700
705
701 #ini = time.time()
706 #ini = time.time()
702
707
703 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
708 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
704 sts = self.procUnitObj.call(opType = opConfObj.type,
709 sts = self.procUnitObj.call(opType = opConfObj.type,
705 opName = opConfObj.name,
710 opName = opConfObj.name,
706 opId = opConfObj.id,
711 opId = opConfObj.id,
707 )
712 )
708
713
709 # total_time = time.time() - ini
714 # total_time = time.time() - ini
710 #
715 #
711 # if total_time > 0.002:
716 # if total_time > 0.002:
712 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
717 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
713
718
714 is_ok = is_ok or sts
719 is_ok = is_ok or sts
715
720
716 return is_ok
721 return is_ok
717
722
718 def close(self):
723 def close(self):
719
724
720 for opConfObj in self.opConfObjList:
725 for opConfObj in self.opConfObjList:
721 if opConfObj.type == 'self':
726 if opConfObj.type == 'self':
722 continue
727 continue
723
728
724 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
729 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
725 opObj.close()
730 opObj.close()
726
731
727 self.procUnitObj.close()
732 self.procUnitObj.close()
728
733
729 return
734 return
730
735
731 class ReadUnitConf(ProcUnitConf):
736 class ReadUnitConf(ProcUnitConf):
732
737
733 path = None
738 path = None
734 startDate = None
739 startDate = None
735 endDate = None
740 endDate = None
736 startTime = None
741 startTime = None
737 endTime = None
742 endTime = None
738
743
739 ELEMENTNAME = 'ReadUnit'
744 ELEMENTNAME = 'ReadUnit'
740
745
741 def __init__(self):
746 def __init__(self):
742
747
743 self.id = None
748 self.id = None
744 self.datatype = None
749 self.datatype = None
745 self.name = None
750 self.name = None
746 self.inputId = None
751 self.inputId = None
747
752
748 self.parentId = None
753 self.parentId = None
749
754
750 self.opConfObjList = []
755 self.opConfObjList = []
751 self.opObjList = []
756 self.opObjList = []
752
757
753 def getElementName(self):
758 def getElementName(self):
754
759
755 return self.ELEMENTNAME
760 return self.ELEMENTNAME
756
761
757 def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, queue=None, **kwargs):
762 def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, queue=None, **kwargs):
758
763
759 #Compatible with old signal chain version
764 #Compatible with old signal chain version
760 if datatype==None and name==None:
765 if datatype==None and name==None:
761 raise ValueError, "datatype or name should be defined"
766 raise ValueError, "datatype or name should be defined"
762
767
763 if name==None:
768 if name==None:
764 if 'Reader' in datatype:
769 if 'Reader' in datatype:
765 name = datatype
770 name = datatype
766 else:
771 else:
767 name = '%sReader' %(datatype)
772 name = '%sReader' %(datatype)
768
773
769 if datatype==None:
774 if datatype==None:
770 datatype = name.replace('Reader','')
775 datatype = name.replace('Reader','')
771
776
772 self.id = id
777 self.id = id
773 self.name = name
778 self.name = name
774 self.datatype = datatype
779 self.datatype = datatype
775
780
776 self.path = os.path.abspath(path)
781 self.path = os.path.abspath(path)
777 self.startDate = startDate
782 self.startDate = startDate
778 self.endDate = endDate
783 self.endDate = endDate
779 self.startTime = startTime
784 self.startTime = startTime
780 self.endTime = endTime
785 self.endTime = endTime
781
786
782 self.inputId = '0'
787 self.inputId = '0'
783 self.parentId = parentId
788 self.parentId = parentId
784 self.queue = queue
789 self.queue = queue
785 self.addRunOperation(**kwargs)
790 self.addRunOperation(**kwargs)
786
791
787 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
792 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
788
793
789 #Compatible with old signal chain version
794 #Compatible with old signal chain version
790 if datatype==None and name==None:
795 if datatype==None and name==None:
791 raise ValueError, "datatype or name should be defined"
796 raise ValueError, "datatype or name should be defined"
792
797
793 if name==None:
798 if name==None:
794 if 'Reader' in datatype:
799 if 'Reader' in datatype:
795 name = datatype
800 name = datatype
796 else:
801 else:
797 name = '%sReader' %(datatype)
802 name = '%sReader' %(datatype)
798
803
799 if datatype==None:
804 if datatype==None:
800 datatype = name.replace('Reader','')
805 datatype = name.replace('Reader','')
801
806
802 self.datatype = datatype
807 self.datatype = datatype
803 self.name = name
808 self.name = name
804 self.path = path
809 self.path = path
805 self.startDate = startDate
810 self.startDate = startDate
806 self.endDate = endDate
811 self.endDate = endDate
807 self.startTime = startTime
812 self.startTime = startTime
808 self.endTime = endTime
813 self.endTime = endTime
809
814
810 self.inputId = '0'
815 self.inputId = '0'
811 self.parentId = parentId
816 self.parentId = parentId
812
817
813 self.updateRunOperation(**kwargs)
818 self.updateRunOperation(**kwargs)
814
819
815 def removeOperations(self):
820 def removeOperations(self):
816
821
817 for obj in self.opConfObjList:
822 for obj in self.opConfObjList:
818 del obj
823 del obj
819
824
820 self.opConfObjList = []
825 self.opConfObjList = []
821
826
822 def addRunOperation(self, **kwargs):
827 def addRunOperation(self, **kwargs):
823
828
824 opObj = self.addOperation(name = 'run', optype = 'self')
829 opObj = self.addOperation(name = 'run', optype = 'self')
825
830
826 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
831 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
827 opObj.addParameter(name='path' , value=self.path, format='str')
832 opObj.addParameter(name='path' , value=self.path, format='str')
828 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
833 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
829 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
834 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
830 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
835 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
831 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
836 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
832 opObj.addParameter(name='queue' , value=self.queue, format='obj')
837 opObj.addParameter(name='queue' , value=self.queue, format='obj')
833
838
834 for key, value in kwargs.items():
839 for key, value in kwargs.items():
835 opObj.addParameter(name=key, value=value, format=type(value).__name__)
840 opObj.addParameter(name=key, value=value, format=type(value).__name__)
836
841
837 return opObj
842 return opObj
838
843
839 def updateRunOperation(self, **kwargs):
844 def updateRunOperation(self, **kwargs):
840
845
841 opObj = self.getOperationObj(name = 'run')
846 opObj = self.getOperationObj(name = 'run')
842 opObj.removeParameters()
847 opObj.removeParameters()
843
848
844 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
849 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
845 opObj.addParameter(name='path' , value=self.path, format='str')
850 opObj.addParameter(name='path' , value=self.path, format='str')
846 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
851 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
847 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
852 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
848 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
853 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
849 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
854 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
850
855
851 for key, value in kwargs.items():
856 for key, value in kwargs.items():
852 opObj.addParameter(name=key, value=value, format=type(value).__name__)
857 opObj.addParameter(name=key, value=value, format=type(value).__name__)
853
858
854 return opObj
859 return opObj
855
860
856 # def makeXml(self, projectElement):
861 # def makeXml(self, projectElement):
857 #
862 #
858 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
863 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
859 # procUnitElement.set('id', str(self.id))
864 # procUnitElement.set('id', str(self.id))
860 # procUnitElement.set('name', self.name)
865 # procUnitElement.set('name', self.name)
861 # procUnitElement.set('datatype', self.datatype)
866 # procUnitElement.set('datatype', self.datatype)
862 # procUnitElement.set('inputId', str(self.inputId))
867 # procUnitElement.set('inputId', str(self.inputId))
863 #
868 #
864 # for opConfObj in self.opConfObjList:
869 # for opConfObj in self.opConfObjList:
865 # opConfObj.makeXml(procUnitElement)
870 # opConfObj.makeXml(procUnitElement)
866
871
867 def readXml(self, upElement):
872 def readXml(self, upElement):
868
873
869 self.id = upElement.get('id')
874 self.id = upElement.get('id')
870 self.name = upElement.get('name')
875 self.name = upElement.get('name')
871 self.datatype = upElement.get('datatype')
876 self.datatype = upElement.get('datatype')
872 self.inputId = upElement.get('inputId')
877 self.inputId = upElement.get('inputId')
873
878
874 if self.ELEMENTNAME == "ReadUnit":
879 if self.ELEMENTNAME == "ReadUnit":
875 self.datatype = self.datatype.replace("Reader", "")
880 self.datatype = self.datatype.replace("Reader", "")
876
881
877 if self.inputId == 'None':
882 if self.inputId == 'None':
878 self.inputId = '0'
883 self.inputId = '0'
879
884
880 self.opConfObjList = []
885 self.opConfObjList = []
881
886
882 opElementList = upElement.iter(OperationConf().getElementName())
887 opElementList = upElement.iter(OperationConf().getElementName())
883
888
884 for opElement in opElementList:
889 for opElement in opElementList:
885 opConfObj = OperationConf()
890 opConfObj = OperationConf()
886 opConfObj.readXml(opElement)
891 opConfObj.readXml(opElement)
887 self.opConfObjList.append(opConfObj)
892 self.opConfObjList.append(opConfObj)
888
893
889 if opConfObj.name == 'run':
894 if opConfObj.name == 'run':
890 self.path = opConfObj.getParameterValue('path')
895 self.path = opConfObj.getParameterValue('path')
891 self.startDate = opConfObj.getParameterValue('startDate')
896 self.startDate = opConfObj.getParameterValue('startDate')
892 self.endDate = opConfObj.getParameterValue('endDate')
897 self.endDate = opConfObj.getParameterValue('endDate')
893 self.startTime = opConfObj.getParameterValue('startTime')
898 self.startTime = opConfObj.getParameterValue('startTime')
894 self.endTime = opConfObj.getParameterValue('endTime')
899 self.endTime = opConfObj.getParameterValue('endTime')
895
900
896 class Project():
901 class Project():
897
902
898 id = None
903 id = None
899 name = None
904 name = None
900 description = None
905 description = None
901 filename = None
906 filename = None
902
907
903 procUnitConfObjDict = None
908 procUnitConfObjDict = None
904
909
905 ELEMENTNAME = 'Project'
910 ELEMENTNAME = 'Project'
906
911
907 plotterQueue = None
912 plotterQueue = None
908
913
909 def __init__(self, plotter_queue=None):
914 def __init__(self, plotter_queue=None):
910
915
911 self.id = None
916 self.id = None
912 self.name = None
917 self.name = None
913 self.description = None
918 self.description = None
914
919
915 self.plotterQueue = plotter_queue
920 self.plotterQueue = plotter_queue
916
921
917 self.procUnitConfObjDict = {}
922 self.procUnitConfObjDict = {}
918
923
919 def __getNewId(self):
924 def __getNewId(self):
920
925
921 idList = self.procUnitConfObjDict.keys()
926 idList = self.procUnitConfObjDict.keys()
922
927
923 id = int(self.id)*10
928 id = int(self.id)*10
924
929
925 while True:
930 while True:
926 id += 1
931 id += 1
927
932
928 if str(id) in idList:
933 if str(id) in idList:
929 continue
934 continue
930
935
931 break
936 break
932
937
933 return str(id)
938 return str(id)
934
939
935 def getElementName(self):
940 def getElementName(self):
936
941
937 return self.ELEMENTNAME
942 return self.ELEMENTNAME
938
943
939 def getId(self):
944 def getId(self):
940
945
941 return self.id
946 return self.id
942
947
943 def updateId(self, new_id):
948 def updateId(self, new_id):
944
949
945 self.id = str(new_id)
950 self.id = str(new_id)
946
951
947 keyList = self.procUnitConfObjDict.keys()
952 keyList = self.procUnitConfObjDict.keys()
948 keyList.sort()
953 keyList.sort()
949
954
950 n = 1
955 n = 1
951 newProcUnitConfObjDict = {}
956 newProcUnitConfObjDict = {}
952
957
953 for procKey in keyList:
958 for procKey in keyList:
954
959
955 procUnitConfObj = self.procUnitConfObjDict[procKey]
960 procUnitConfObj = self.procUnitConfObjDict[procKey]
956 idProcUnit = str(int(self.id)*10 + n)
961 idProcUnit = str(int(self.id)*10 + n)
957 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
962 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
958
963
959 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
964 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
960 n += 1
965 n += 1
961
966
962 self.procUnitConfObjDict = newProcUnitConfObjDict
967 self.procUnitConfObjDict = newProcUnitConfObjDict
963
968
964 def setup(self, id, name, description):
969 def setup(self, id, name, description):
965
970
966 self.id = str(id)
971 self.id = str(id)
967 self.name = name
972 self.name = name
968 self.description = description
973 self.description = description
969
974
970 def update(self, name, description):
975 def update(self, name, description):
971
976
972 self.name = name
977 self.name = name
973 self.description = description
978 self.description = description
974
979
975 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
980 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
976
981
977 if id is None:
982 if id is None:
978 idReadUnit = self.__getNewId()
983 idReadUnit = self.__getNewId()
979 else:
984 else:
980 idReadUnit = str(id)
985 idReadUnit = str(id)
981
986
982 readUnitConfObj = ReadUnitConf()
987 readUnitConfObj = ReadUnitConf()
983 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
988 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
984
989
985 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
990 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
986
991
987 return readUnitConfObj
992 return readUnitConfObj
988
993
989 def addProcUnit(self, inputId='0', datatype=None, name=None):
994 def addProcUnit(self, inputId='0', datatype=None, name=None):
990
995
991 idProcUnit = self.__getNewId()
996 idProcUnit = self.__getNewId()
992
997
993 procUnitConfObj = ProcUnitConf()
998 procUnitConfObj = ProcUnitConf()
994 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
999 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
995
1000
996 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1001 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
997
1002
998 return procUnitConfObj
1003 return procUnitConfObj
999
1004
1000 def removeProcUnit(self, id):
1005 def removeProcUnit(self, id):
1001
1006
1002 if id in self.procUnitConfObjDict.keys():
1007 if id in self.procUnitConfObjDict.keys():
1003 self.procUnitConfObjDict.pop(id)
1008 self.procUnitConfObjDict.pop(id)
1004
1009
1005 def getReadUnitId(self):
1010 def getReadUnitId(self):
1006
1011
1007 readUnitConfObj = self.getReadUnitObj()
1012 readUnitConfObj = self.getReadUnitObj()
1008
1013
1009 return readUnitConfObj.id
1014 return readUnitConfObj.id
1010
1015
1011 def getReadUnitObj(self):
1016 def getReadUnitObj(self):
1012
1017
1013 for obj in self.procUnitConfObjDict.values():
1018 for obj in self.procUnitConfObjDict.values():
1014 if obj.getElementName() == "ReadUnit":
1019 if obj.getElementName() == "ReadUnit":
1015 return obj
1020 return obj
1016
1021
1017 return None
1022 return None
1018
1023
1019 def getProcUnitObj(self, id=None, name=None):
1024 def getProcUnitObj(self, id=None, name=None):
1020
1025
1021 if id != None:
1026 if id != None:
1022 return self.procUnitConfObjDict[id]
1027 return self.procUnitConfObjDict[id]
1023
1028
1024 if name != None:
1029 if name != None:
1025 return self.getProcUnitObjByName(name)
1030 return self.getProcUnitObjByName(name)
1026
1031
1027 return None
1032 return None
1028
1033
1029 def getProcUnitObjByName(self, name):
1034 def getProcUnitObjByName(self, name):
1030
1035
1031 for obj in self.procUnitConfObjDict.values():
1036 for obj in self.procUnitConfObjDict.values():
1032 if obj.name == name:
1037 if obj.name == name:
1033 return obj
1038 return obj
1034
1039
1035 return None
1040 return None
1036
1041
1037 def procUnitItems(self):
1042 def procUnitItems(self):
1038
1043
1039 return self.procUnitConfObjDict.items()
1044 return self.procUnitConfObjDict.items()
1040
1045
1041 def makeXml(self):
1046 def makeXml(self):
1042
1047
1043 projectElement = Element('Project')
1048 projectElement = Element('Project')
1044 projectElement.set('id', str(self.id))
1049 projectElement.set('id', str(self.id))
1045 projectElement.set('name', self.name)
1050 projectElement.set('name', self.name)
1046 projectElement.set('description', self.description)
1051 projectElement.set('description', self.description)
1047
1052
1048 for procUnitConfObj in self.procUnitConfObjDict.values():
1053 for procUnitConfObj in self.procUnitConfObjDict.values():
1049 procUnitConfObj.makeXml(projectElement)
1054 procUnitConfObj.makeXml(projectElement)
1050
1055
1051 self.projectElement = projectElement
1056 self.projectElement = projectElement
1052
1057
1053 def writeXml(self, filename=None):
1058 def writeXml(self, filename=None):
1054
1059
1055 if filename == None:
1060 if filename == None:
1056 if self.filename:
1061 if self.filename:
1057 filename = self.filename
1062 filename = self.filename
1058 else:
1063 else:
1059 filename = "schain.xml"
1064 filename = "schain.xml"
1060
1065
1061 if not filename:
1066 if not filename:
1062 print "filename has not been defined. Use setFilename(filename) for do it."
1067 print "filename has not been defined. Use setFilename(filename) for do it."
1063 return 0
1068 return 0
1064
1069
1065 abs_file = os.path.abspath(filename)
1070 abs_file = os.path.abspath(filename)
1066
1071
1067 if not os.access(os.path.dirname(abs_file), os.W_OK):
1072 if not os.access(os.path.dirname(abs_file), os.W_OK):
1068 print "No write permission on %s" %os.path.dirname(abs_file)
1073 print "No write permission on %s" %os.path.dirname(abs_file)
1069 return 0
1074 return 0
1070
1075
1071 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1076 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1072 print "File %s already exists and it could not be overwriten" %abs_file
1077 print "File %s already exists and it could not be overwriten" %abs_file
1073 return 0
1078 return 0
1074
1079
1075 self.makeXml()
1080 self.makeXml()
1076
1081
1077 ElementTree(self.projectElement).write(abs_file, method='xml')
1082 ElementTree(self.projectElement).write(abs_file, method='xml')
1078
1083
1079 self.filename = abs_file
1084 self.filename = abs_file
1080
1085
1081 return 1
1086 return 1
1082
1087
1083 def readXml(self, filename = None):
1088 def readXml(self, filename = None):
1084
1089
1085 if not filename:
1090 if not filename:
1086 print "filename is not defined"
1091 print "filename is not defined"
1087 return 0
1092 return 0
1088
1093
1089 abs_file = os.path.abspath(filename)
1094 abs_file = os.path.abspath(filename)
1090
1095
1091 if not os.path.isfile(abs_file):
1096 if not os.path.isfile(abs_file):
1092 print "%s file does not exist" %abs_file
1097 print "%s file does not exist" %abs_file
1093 return 0
1098 return 0
1094
1099
1095 self.projectElement = None
1100 self.projectElement = None
1096 self.procUnitConfObjDict = {}
1101 self.procUnitConfObjDict = {}
1097
1102
1098 try:
1103 try:
1099 self.projectElement = ElementTree().parse(abs_file)
1104 self.projectElement = ElementTree().parse(abs_file)
1100 except:
1105 except:
1101 print "Error reading %s, verify file format" %filename
1106 print "Error reading %s, verify file format" %filename
1102 return 0
1107 return 0
1103
1108
1104 self.project = self.projectElement.tag
1109 self.project = self.projectElement.tag
1105
1110
1106 self.id = self.projectElement.get('id')
1111 self.id = self.projectElement.get('id')
1107 self.name = self.projectElement.get('name')
1112 self.name = self.projectElement.get('name')
1108 self.description = self.projectElement.get('description')
1113 self.description = self.projectElement.get('description')
1109
1114
1110 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1115 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1111
1116
1112 for readUnitElement in readUnitElementList:
1117 for readUnitElement in readUnitElementList:
1113 readUnitConfObj = ReadUnitConf()
1118 readUnitConfObj = ReadUnitConf()
1114 readUnitConfObj.readXml(readUnitElement)
1119 readUnitConfObj.readXml(readUnitElement)
1115
1120
1116 if readUnitConfObj.parentId == None:
1121 if readUnitConfObj.parentId == None:
1117 readUnitConfObj.parentId = self.id
1122 readUnitConfObj.parentId = self.id
1118
1123
1119 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1124 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1120
1125
1121 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1126 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1122
1127
1123 for procUnitElement in procUnitElementList:
1128 for procUnitElement in procUnitElementList:
1124 procUnitConfObj = ProcUnitConf()
1129 procUnitConfObj = ProcUnitConf()
1125 procUnitConfObj.readXml(procUnitElement)
1130 procUnitConfObj.readXml(procUnitElement)
1126
1131
1127 if procUnitConfObj.parentId == None:
1132 if procUnitConfObj.parentId == None:
1128 procUnitConfObj.parentId = self.id
1133 procUnitConfObj.parentId = self.id
1129
1134
1130 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1135 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1131
1136
1132 self.filename = abs_file
1137 self.filename = abs_file
1133
1138
1134 return 1
1139 return 1
1135
1140
1136 def printattr(self):
1141 def printattr(self):
1137
1142
1138 print "Project[%s]: name = %s, description = %s" %(self.id,
1143 print "Project[%s]: name = %s, description = %s" %(self.id,
1139 self.name,
1144 self.name,
1140 self.description)
1145 self.description)
1141
1146
1142 for procUnitConfObj in self.procUnitConfObjDict.values():
1147 for procUnitConfObj in self.procUnitConfObjDict.values():
1143 procUnitConfObj.printattr()
1148 procUnitConfObj.printattr()
1144
1149
1145 def createObjects(self):
1150 def createObjects(self):
1146
1151
1147 for procUnitConfObj in self.procUnitConfObjDict.values():
1152 for procUnitConfObj in self.procUnitConfObjDict.values():
1148 procUnitConfObj.createObjects(self.plotterQueue)
1153 procUnitConfObj.createObjects(self.plotterQueue)
1149
1154
1150 def __connect(self, objIN, thisObj):
1155 def __connect(self, objIN, thisObj):
1151
1156
1152 thisObj.setInput(objIN.getOutputObj())
1157 thisObj.setInput(objIN.getOutputObj())
1153
1158
1154 def connectObjects(self):
1159 def connectObjects(self):
1155
1160
1156 for thisPUConfObj in self.procUnitConfObjDict.values():
1161 for thisPUConfObj in self.procUnitConfObjDict.values():
1157
1162
1158 inputId = thisPUConfObj.getInputId()
1163 inputId = thisPUConfObj.getInputId()
1159
1164
1160 if int(inputId) == 0:
1165 if int(inputId) == 0:
1161 continue
1166 continue
1162
1167
1163 #Get input object
1168 #Get input object
1164 puConfINObj = self.procUnitConfObjDict[inputId]
1169 puConfINObj = self.procUnitConfObjDict[inputId]
1165 puObjIN = puConfINObj.getProcUnitObj()
1170 puObjIN = puConfINObj.getProcUnitObj()
1166
1171
1167 #Get current object
1172 #Get current object
1168 thisPUObj = thisPUConfObj.getProcUnitObj()
1173 thisPUObj = thisPUConfObj.getProcUnitObj()
1169
1174
1170 self.__connect(puObjIN, thisPUObj)
1175 self.__connect(puObjIN, thisPUObj)
1171
1176
1172 def __handleError(self, procUnitConfObj, send_email=True):
1177 def __handleError(self, procUnitConfObj, send_email=True):
1173
1178
1174 import socket
1179 import socket
1175
1180
1176 err = traceback.format_exception(sys.exc_info()[0],
1181 err = traceback.format_exception(sys.exc_info()[0],
1177 sys.exc_info()[1],
1182 sys.exc_info()[1],
1178 sys.exc_info()[2])
1183 sys.exc_info()[2])
1179
1184
1180 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1185 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1181 print "***** %s" %err[-1]
1186 print "***** %s" %err[-1]
1182
1187
1183 message = "".join(err)
1188 message = "".join(err)
1184
1189
1185 sys.stderr.write(message)
1190 sys.stderr.write(message)
1186
1191
1187 if not send_email:
1192 if not send_email:
1188 return
1193 return
1189
1194
1190 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1195 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1191
1196
1192 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1197 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1193 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1198 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1194 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1199 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1195 subtitle += "Configuration file: %s\n" %self.filename
1200 subtitle += "Configuration file: %s\n" %self.filename
1196 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1201 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1197
1202
1198 readUnitConfObj = self.getReadUnitObj()
1203 readUnitConfObj = self.getReadUnitObj()
1199 if readUnitConfObj:
1204 if readUnitConfObj:
1200 subtitle += "\nInput parameters:\n"
1205 subtitle += "\nInput parameters:\n"
1201 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1206 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1202 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1207 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1203 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1208 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1204 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1209 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1205 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1210 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1206 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1211 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1207
1212
1208 adminObj = schainpy.admin.SchainNotify()
1213 adminObj = schainpy.admin.SchainNotify()
1209 adminObj.sendAlert(message=message,
1214 adminObj.sendAlert(message=message,
1210 subject=subject,
1215 subject=subject,
1211 subtitle=subtitle,
1216 subtitle=subtitle,
1212 filename=self.filename)
1217 filename=self.filename)
1213
1218
1214 def isPaused(self):
1219 def isPaused(self):
1215 return 0
1220 return 0
1216
1221
1217 def isStopped(self):
1222 def isStopped(self):
1218 return 0
1223 return 0
1219
1224
1220 def runController(self):
1225 def runController(self):
1221 """
1226 """
1222 returns 0 when this process has been stopped, 1 otherwise
1227 returns 0 when this process has been stopped, 1 otherwise
1223 """
1228 """
1224
1229
1225 if self.isPaused():
1230 if self.isPaused():
1226 print "Process suspended"
1231 print "Process suspended"
1227
1232
1228 while True:
1233 while True:
1229 sleep(0.1)
1234 sleep(0.1)
1230
1235
1231 if not self.isPaused():
1236 if not self.isPaused():
1232 break
1237 break
1233
1238
1234 if self.isStopped():
1239 if self.isStopped():
1235 break
1240 break
1236
1241
1237 print "Process reinitialized"
1242 print "Process reinitialized"
1238
1243
1239 if self.isStopped():
1244 if self.isStopped():
1240 print "Process stopped"
1245 print "Process stopped"
1241 return 0
1246 return 0
1242
1247
1243 return 1
1248 return 1
1244
1249
1245 def setFilename(self, filename):
1250 def setFilename(self, filename):
1246
1251
1247 self.filename = filename
1252 self.filename = filename
1248
1253
1249 def setPlotterQueue(self, plotter_queue):
1254 def setPlotterQueue(self, plotter_queue):
1250
1255
1251 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1256 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1252
1257
1253 def getPlotterQueue(self):
1258 def getPlotterQueue(self):
1254
1259
1255 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1260 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1256
1261
1257 def useExternalPlotter(self):
1262 def useExternalPlotter(self):
1258
1263
1259 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1264 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1260
1265
1261 def run(self):
1266 def run(self):
1262
1267
1263 print
1268 print
1264 print "*"*60
1269 print "*"*60
1265 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1270 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1266 print "*"*60
1271 print "*"*60
1267 print
1272 print
1268
1273
1269 keyList = self.procUnitConfObjDict.keys()
1274 keyList = self.procUnitConfObjDict.keys()
1270 keyList.sort()
1275 keyList.sort()
1271
1276
1272 while(True):
1277 while(True):
1273
1278
1274 is_ok = False
1279 is_ok = False
1275
1280
1276 for procKey in keyList:
1281 for procKey in keyList:
1277 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1282 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1278
1283
1279 procUnitConfObj = self.procUnitConfObjDict[procKey]
1284 procUnitConfObj = self.procUnitConfObjDict[procKey]
1280
1285
1281 try:
1286 try:
1282 sts = procUnitConfObj.run()
1287 sts = procUnitConfObj.run()
1283 is_ok = is_ok or sts
1288 is_ok = is_ok or sts
1284 except KeyboardInterrupt:
1289 except KeyboardInterrupt:
1285 is_ok = False
1290 is_ok = False
1286 break
1291 break
1287 except ValueError, e:
1292 except ValueError, e:
1288 sleep(0.5)
1293 sleep(0.5)
1289 self.__handleError(procUnitConfObj, send_email=True)
1294 self.__handleError(procUnitConfObj, send_email=True)
1290 is_ok = False
1295 is_ok = False
1291 break
1296 break
1292 except:
1297 except:
1293 sleep(0.5)
1298 sleep(0.5)
1294 self.__handleError(procUnitConfObj)
1299 self.__handleError(procUnitConfObj)
1295 is_ok = False
1300 is_ok = False
1296 break
1301 break
1297
1302
1298 #If every process unit finished so end process
1303 #If every process unit finished so end process
1299 if not(is_ok):
1304 if not(is_ok):
1300 # print "Every process unit have finished"
1305 # print "Every process unit have finished"
1301 break
1306 break
1302
1307
1303 if not self.runController():
1308 if not self.runController():
1304 break
1309 break
1305
1310
1306 #Closing every process
1311 #Closing every process
1307 for procKey in keyList:
1312 for procKey in keyList:
1308 procUnitConfObj = self.procUnitConfObjDict[procKey]
1313 procUnitConfObj = self.procUnitConfObjDict[procKey]
1309 procUnitConfObj.close()
1314 procUnitConfObj.close()
1310
1315
1311 print "Process finished"
1316 print "Process finished"
1312
1317
1313 def start(self):
1318 def start(self):
1314
1319
1315 self.writeXml()
1320 self.writeXml()
1316 self.createObjects()
1321 self.createObjects()
1317 self.connectObjects()
1322 self.connectObjects()
1318 self.run()
1323 self.run()
Show file before | Show file after | Hide comments
@@ -1,377 +1,378
1
1
2 import os
2 import os
3 import zmq
3 import zmq
4 import time
4 import time
5 import numpy
5 import numpy
6 import datetime
6 import datetime
7 import numpy as np
7 import numpy as np
8 import matplotlib.pyplot as plt
8 import matplotlib.pyplot as plt
9 from mpl_toolkits.axes_grid1 import make_axes_locatable
9 from mpl_toolkits.axes_grid1 import make_axes_locatable
10 from matplotlib.ticker import FuncFormatter, LinearLocator
10 from matplotlib.ticker import FuncFormatter, LinearLocator
11 from multiprocessing import Process
11 from multiprocessing import Process
12
12
13 from schainpy.model.proc.jroproc_base import Operation
13 from schainpy.model.proc.jroproc_base import Operation
14
14
15 #plt.ion()
15 #plt.ion()
16
16
17 func = lambda x, pos: ('%s') %(datetime.datetime.utcfromtimestamp(x).strftime('%H:%M'))
17 func = lambda x, pos: ('%s') %(datetime.datetime.utcfromtimestamp(x).strftime('%H:%M'))
18
18
19 d1970 = datetime.datetime(1970,1,1)
19 d1970 = datetime.datetime(1970,1,1)
20
20
21 class PlotData(Operation, Process):
21 class PlotData(Operation, Process):
22
22
23 CODE = 'Figure'
23 CODE = 'Figure'
24 colormap = 'jet'
24 colormap = 'jet'
25 CONFLATE = True
25 CONFLATE = True
26 __MAXNUMX = 80
26 __MAXNUMX = 80
27 __MAXNUMY = 80
27 __MAXNUMY = 80
28 __missing = 1E30
28 __missing = 1E30
29
29
30 def __init__(self, **kwargs):
30 def __init__(self, **kwargs):
31
31
32 Operation.__init__(self, **kwargs)
32 Operation.__init__(self, plot=True, **kwargs)
33 Process.__init__(self)
33 Process.__init__(self)
34 self.kwargs['code'] = self.CODE
34 self.mp = False
35 self.mp = False
35 self.dataOut = None
36 self.dataOut = None
36 self.isConfig = False
37 self.isConfig = False
37 self.figure = None
38 self.figure = None
38 self.axes = []
39 self.axes = []
39 self.localtime = kwargs.pop('localtime', True)
40 self.localtime = kwargs.pop('localtime', True)
40 self.show = kwargs.get('show', True)
41 self.show = kwargs.get('show', True)
41 self.save = kwargs.get('save', False)
42 self.save = kwargs.get('save', False)
42 self.colormap = kwargs.get('colormap', self.colormap)
43 self.colormap = kwargs.get('colormap', self.colormap)
43 self.showprofile = kwargs.get('showprofile', False)
44 self.showprofile = kwargs.get('showprofile', False)
44 self.title = kwargs.get('wintitle', '')
45 self.title = kwargs.get('wintitle', '')
45 self.xaxis = kwargs.get('xaxis', 'time')
46 self.xaxis = kwargs.get('xaxis', 'time')
46 self.zmin = kwargs.get('zmin', None)
47 self.zmin = kwargs.get('zmin', None)
47 self.zmax = kwargs.get('zmax', None)
48 self.zmax = kwargs.get('zmax', None)
48 self.xmin = kwargs.get('xmin', None)
49 self.xmin = kwargs.get('xmin', None)
49 self.xmax = kwargs.get('xmax', None)
50 self.xmax = kwargs.get('xmax', None)
50 self.xrange = kwargs.get('xrange', 24)
51 self.xrange = kwargs.get('xrange', 24)
51 self.ymin = kwargs.get('ymin', None)
52 self.ymin = kwargs.get('ymin', None)
52 self.ymax = kwargs.get('ymax', None)
53 self.ymax = kwargs.get('ymax', None)
53 self.throttle_value = 1
54 self.throttle_value = 1
54 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
55 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
55
56
56 if x_buffer.shape[0] < 2:
57 if x_buffer.shape[0] < 2:
57 return x_buffer, y_buffer, z_buffer
58 return x_buffer, y_buffer, z_buffer
58
59
59 deltas = x_buffer[1:] - x_buffer[0:-1]
60 deltas = x_buffer[1:] - x_buffer[0:-1]
60 x_median = np.median(deltas)
61 x_median = np.median(deltas)
61
62
62 index = np.where(deltas > 5*x_median)
63 index = np.where(deltas > 5*x_median)
63
64
64 if len(index[0]) != 0:
65 if len(index[0]) != 0:
65 z_buffer[::, index[0], ::] = self.__missing
66 z_buffer[::, index[0], ::] = self.__missing
66 z_buffer = np.ma.masked_inside(z_buffer,
67 z_buffer = np.ma.masked_inside(z_buffer,
67 0.99*self.__missing,
68 0.99*self.__missing,
68 1.01*self.__missing)
69 1.01*self.__missing)
69
70
70 return x_buffer, y_buffer, z_buffer
71 return x_buffer, y_buffer, z_buffer
71
72
72 def decimate(self):
73 def decimate(self):
73
74
74 # dx = int(len(self.x)/self.__MAXNUMX) + 1
75 # dx = int(len(self.x)/self.__MAXNUMX) + 1
75 dy = int(len(self.y)/self.__MAXNUMY) + 1
76 dy = int(len(self.y)/self.__MAXNUMY) + 1
76
77
77 # x = self.x[::dx]
78 # x = self.x[::dx]
78 x = self.x
79 x = self.x
79 y = self.y[::dy]
80 y = self.y[::dy]
80 z = self.z[::, ::, ::dy]
81 z = self.z[::, ::, ::dy]
81
82
82 return x, y, z
83 return x, y, z
83
84
84 def __plot(self):
85 def __plot(self):
85
86
86 print 'plotting...{}'.format(self.CODE)
87 print 'plotting...{}'.format(self.CODE)
87
88
88 self.plot()
89 self.plot()
89 self.figure.suptitle('{} {} - Date:{}'.format(self.title, self.CODE.upper(),
90 self.figure.suptitle('{} {} - Date:{}'.format(self.title, self.CODE.upper(),
90 datetime.datetime.utcfromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')))
91 datetime.datetime.utcfromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')))
91
92
92 if self.save:
93 if self.save:
93 figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,
94 figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,
94 datetime.datetime.utcfromtimestamp(self.times[0]).strftime('%y%m%d_%H%M%S')))
95 datetime.datetime.utcfromtimestamp(self.times[0]).strftime('%y%m%d_%H%M%S')))
95 print 'Saving figure: {}'.format(figname)
96 print 'Saving figure: {}'.format(figname)
96 self.figure.savefig(figname)
97 self.figure.savefig(figname)
97
98
98 self.figure.canvas.draw()
99 self.figure.canvas.draw()
99
100
100 def plot(self):
101 def plot(self):
101
102
102 print 'plotting...{}'.format(self.CODE.upper())
103 print 'plotting...{}'.format(self.CODE.upper())
103 return
104 return
104
105
105 def run(self):
106 def run(self):
106
107
107 print '[Starting] {}'.format(self.name)
108 print '[Starting] {}'.format(self.name)
108 context = zmq.Context()
109 context = zmq.Context()
109 receiver = context.socket(zmq.SUB)
110 receiver = context.socket(zmq.SUB)
110 receiver.setsockopt(zmq.SUBSCRIBE, '')
111 receiver.setsockopt(zmq.SUBSCRIBE, '')
111 receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
112 receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
112 receiver.connect("ipc:///tmp/zmq.plots")
113 receiver.connect("ipc:///tmp/zmq.plots")
113
114
114 while True:
115 while True:
115 try:
116 try:
116 #if True:
117 #if True:
117 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
118 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
118 self.dataOut = self.data['dataOut']
119 self.dataOut = self.data['dataOut']
119 self.times = self.data['times']
120 self.times = self.data['times']
120 self.times.sort()
121 self.times.sort()
121 self.throttle_value = self.data['throttle']
122 self.throttle_value = self.data['throttle']
122 self.min_time = self.times[0]
123 self.min_time = self.times[0]
123 self.max_time = self.times[-1]
124 self.max_time = self.times[-1]
124
125
125 if self.isConfig is False:
126 if self.isConfig is False:
126 self.setup()
127 self.setup()
127 self.isConfig = True
128 self.isConfig = True
128 self.__plot()
129 self.__plot()
129
130
130 if self.data['ENDED'] is True:
131 if self.data['ENDED'] is True:
131 # self.__plot()
132 # self.__plot()
132 self.isConfig = False
133 self.isConfig = False
133
134
134 except zmq.Again as e:
135 except zmq.Again as e:
135 print 'Waiting for data...'
136 print 'Waiting for data...'
136 plt.pause(self.throttle_value)
137 plt.pause(self.throttle_value)
137 # time.sleep(3)
138 # time.sleep(3)
138
139
139 def close(self):
140 def close(self):
140 if self.dataOut:
141 if self.dataOut:
141 self._plot()
142 self._plot()
142
143
143
144
144 class PlotSpectraData(PlotData):
145 class PlotSpectraData(PlotData):
145
146
146 CODE = 'spc'
147 CODE = 'spc'
147 colormap = 'jro'
148 colormap = 'jro'
148 CONFLATE = False
149 CONFLATE = False
149 def setup(self):
150 def setup(self):
150
151
151 ncolspan = 1
152 ncolspan = 1
152 colspan = 1
153 colspan = 1
153 self.ncols = int(numpy.sqrt(self.dataOut.nChannels)+0.9)
154 self.ncols = int(numpy.sqrt(self.dataOut.nChannels)+0.9)
154 self.nrows = int(self.dataOut.nChannels*1./self.ncols + 0.9)
155 self.nrows = int(self.dataOut.nChannels*1./self.ncols + 0.9)
155 self.width = 3.6*self.ncols
156 self.width = 3.6*self.ncols
156 self.height = 3.2*self.nrows
157 self.height = 3.2*self.nrows
157 if self.showprofile:
158 if self.showprofile:
158 ncolspan = 3
159 ncolspan = 3
159 colspan = 2
160 colspan = 2
160 self.width += 1.2*self.ncols
161 self.width += 1.2*self.ncols
161
162
162 self.ylabel = 'Range [Km]'
163 self.ylabel = 'Range [Km]'
163 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
164 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
164
165
165 if self.figure is None:
166 if self.figure is None:
166 self.figure = plt.figure(figsize=(self.width, self.height),
167 self.figure = plt.figure(figsize=(self.width, self.height),
167 edgecolor='k',
168 edgecolor='k',
168 facecolor='w')
169 facecolor='w')
169 else:
170 else:
170 self.figure.clf()
171 self.figure.clf()
171
172
172 n = 0
173 n = 0
173 for y in range(self.nrows):
174 for y in range(self.nrows):
174 for x in range(self.ncols):
175 for x in range(self.ncols):
175 if n >= self.dataOut.nChannels:
176 if n >= self.dataOut.nChannels:
176 break
177 break
177 ax = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan), 1, colspan)
178 ax = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan), 1, colspan)
178 if self.showprofile:
179 if self.showprofile:
179 ax.ax_profile = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan+colspan), 1, 1)
180 ax.ax_profile = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan+colspan), 1, 1)
180
181
181 ax.firsttime = True
182 ax.firsttime = True
182 self.axes.append(ax)
183 self.axes.append(ax)
183 n += 1
184 n += 1
184
185
185 self.figure.subplots_adjust(wspace=0.9, hspace=0.5)
186 self.figure.subplots_adjust(wspace=0.9, hspace=0.5)
186 self.figure.show()
187 self.figure.show()
187
188
188 def plot(self):
189 def plot(self):
189
190
190 if self.xaxis == "frequency":
191 if self.xaxis == "frequency":
191 x = self.dataOut.getFreqRange(1)/1000.
192 x = self.dataOut.getFreqRange(1)/1000.
192 xlabel = "Frequency (kHz)"
193 xlabel = "Frequency (kHz)"
193 elif self.xaxis == "time":
194 elif self.xaxis == "time":
194 x = self.dataOut.getAcfRange(1)
195 x = self.dataOut.getAcfRange(1)
195 xlabel = "Time (ms)"
196 xlabel = "Time (ms)"
196 else:
197 else:
197 x = self.dataOut.getVelRange(1)
198 x = self.dataOut.getVelRange(1)
198 xlabel = "Velocity (m/s)"
199 xlabel = "Velocity (m/s)"
199
200
200 y = self.dataOut.getHeiRange()
201 y = self.dataOut.getHeiRange()
201 z = self.data[self.CODE]
202 z = self.data[self.CODE]
202
203
203 for n, ax in enumerate(self.axes):
204 for n, ax in enumerate(self.axes):
204
205
205 if ax.firsttime:
206 if ax.firsttime:
206 self.xmax = self.xmax if self.xmax else np.nanmax(x)
207 self.xmax = self.xmax if self.xmax else np.nanmax(x)
207 self.xmin = self.xmin if self.xmin else -self.xmax
208 self.xmin = self.xmin if self.xmin else -self.xmax
208 self.ymin = self.ymin if self.ymin else np.nanmin(y)
209 self.ymin = self.ymin if self.ymin else np.nanmin(y)
209 self.ymax = self.ymax if self.ymax else np.nanmax(y)
210 self.ymax = self.ymax if self.ymax else np.nanmax(y)
210 self.zmin = self.zmin if self.zmin else np.nanmin(z)
211 self.zmin = self.zmin if self.zmin else np.nanmin(z)
211 self.zmax = self.zmax if self.zmax else np.nanmax(z)
212 self.zmax = self.zmax if self.zmax else np.nanmax(z)
212 ax.plot = ax.pcolormesh(x, y, z[n].T,
213 ax.plot = ax.pcolormesh(x, y, z[n].T,
213 vmin=self.zmin,
214 vmin=self.zmin,
214 vmax=self.zmax,
215 vmax=self.zmax,
215 cmap=plt.get_cmap(self.colormap)
216 cmap=plt.get_cmap(self.colormap)
216 )
217 )
217 divider = make_axes_locatable(ax)
218 divider = make_axes_locatable(ax)
218 cax = divider.new_horizontal(size='3%', pad=0.05)
219 cax = divider.new_horizontal(size='3%', pad=0.05)
219 self.figure.add_axes(cax)
220 self.figure.add_axes(cax)
220 plt.colorbar(ax.plot, cax)
221 plt.colorbar(ax.plot, cax)
221
222
222 ax.set_xlim(self.xmin, self.xmax)
223 ax.set_xlim(self.xmin, self.xmax)
223 ax.set_ylim(self.ymin, self.ymax)
224 ax.set_ylim(self.ymin, self.ymax)
224
225
225 ax.xaxis.set_major_locator(LinearLocator(5))
226 ax.xaxis.set_major_locator(LinearLocator(5))
226 #ax.yaxis.set_major_locator(LinearLocator(4))
227 #ax.yaxis.set_major_locator(LinearLocator(4))
227
228
228 ax.set_ylabel(self.ylabel)
229 ax.set_ylabel(self.ylabel)
229 ax.set_xlabel(xlabel)
230 ax.set_xlabel(xlabel)
230
231
231 ax.firsttime = False
232 ax.firsttime = False
232
233
233 if self.showprofile:
234 if self.showprofile:
234 ax.plot_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0]
235 ax.plot_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0]
235 ax.ax_profile.set_xlim(self.zmin, self.zmax)
236 ax.ax_profile.set_xlim(self.zmin, self.zmax)
236 ax.ax_profile.set_ylim(self.ymin, self.ymax)
237 ax.ax_profile.set_ylim(self.ymin, self.ymax)
237 ax.ax_profile.set_xlabel('dB')
238 ax.ax_profile.set_xlabel('dB')
238 ax.ax_profile.grid(b=True, axis='x')
239 ax.ax_profile.grid(b=True, axis='x')
239 ax.plot_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y,
240 ax.plot_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y,
240 color="k", linestyle="dashed", lw=2)[0]
241 color="k", linestyle="dashed", lw=2)[0]
241 [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()]
242 [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()]
242 else:
243 else:
243 ax.plot.set_array(z[n].T.ravel())
244 ax.plot.set_array(z[n].T.ravel())
244 if self.showprofile:
245 if self.showprofile:
245 ax.plot_profile.set_data(self.data['rti'][self.max_time][n], y)
246 ax.plot_profile.set_data(self.data['rti'][self.max_time][n], y)
246 ax.plot_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y)
247 ax.plot_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y)
247
248
248 ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]),
249 ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]),
249 size=8)
250 size=8)
250
251
251 class PlotRTIData(PlotData):
252 class PlotRTIData(PlotData):
252
253
253 CODE = 'rti'
254 CODE = 'rti'
254 colormap = 'jro'
255 colormap = 'jro'
255
256
256 def setup(self):
257 def setup(self):
257 self.ncols = 1
258 self.ncols = 1
258 self.nrows = self.dataOut.nChannels
259 self.nrows = self.dataOut.nChannels
259 self.width = 10
260 self.width = 10
260 self.height = 2.2*self.nrows
261 self.height = 2.2*self.nrows
261 self.ylabel = 'Range [Km]'
262 self.ylabel = 'Range [Km]'
262 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
263 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
263
264
264 if self.figure is None:
265 if self.figure is None:
265 self.figure = plt.figure(figsize=(self.width, self.height),
266 self.figure = plt.figure(figsize=(self.width, self.height),
266 edgecolor='k',
267 edgecolor='k',
267 facecolor='w')
268 facecolor='w')
268 else:
269 else:
269 self.figure.clf()
270 self.figure.clf()
270 self.axes = []
271 self.axes = []
271
272
272 for n in range(self.nrows):
273 for n in range(self.nrows):
273 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
274 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
274 ax.firsttime = True
275 ax.firsttime = True
275 self.axes.append(ax)
276 self.axes.append(ax)
276 self.figure.subplots_adjust(hspace=0.5)
277 self.figure.subplots_adjust(hspace=0.5)
277 self.figure.show()
278 self.figure.show()
278
279
279 def plot(self):
280 def plot(self):
280
281
281 self.x = np.array(self.times)
282 self.x = np.array(self.times)
282 self.y = self.dataOut.getHeiRange()
283 self.y = self.dataOut.getHeiRange()
283 self.z = []
284 self.z = []
284
285
285 for ch in range(self.nrows):
286 for ch in range(self.nrows):
286 self.z.append([self.data[self.CODE][t][ch] for t in self.times])
287 self.z.append([self.data[self.CODE][t][ch] for t in self.times])
287
288
288 self.z = np.array(self.z)
289 self.z = np.array(self.z)
289 for n, ax in enumerate(self.axes):
290 for n, ax in enumerate(self.axes):
290
291
291 x, y, z = self.fill_gaps(*self.decimate())
292 x, y, z = self.fill_gaps(*self.decimate())
292 xmin = self.min_time
293 xmin = self.min_time
293 xmax = xmin+self.xrange*60*60
294 xmax = xmin+self.xrange*60*60
294 if ax.firsttime:
295 if ax.firsttime:
295 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
296 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
296 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
297 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
297 self.zmin = self.zmin if self.zmin else np.nanmin(self.z)
298 self.zmin = self.zmin if self.zmin else np.nanmin(self.z)
298 self.zmax = self.zmax if self.zmax else np.nanmax(self.z)
299 self.zmax = self.zmax if self.zmax else np.nanmax(self.z)
299 plot = ax.pcolormesh(x, y, z[n].T,
300 plot = ax.pcolormesh(x, y, z[n].T,
300 vmin=self.zmin,
301 vmin=self.zmin,
301 vmax=self.zmax,
302 vmax=self.zmax,
302 cmap=plt.get_cmap(self.colormap)
303 cmap=plt.get_cmap(self.colormap)
303 )
304 )
304 divider = make_axes_locatable(ax)
305 divider = make_axes_locatable(ax)
305 cax = divider.new_horizontal(size='2%', pad=0.05)
306 cax = divider.new_horizontal(size='2%', pad=0.05)
306 self.figure.add_axes(cax)
307 self.figure.add_axes(cax)
307 plt.colorbar(plot, cax)
308 plt.colorbar(plot, cax)
308 ax.set_ylim(self.ymin, self.ymax)
309 ax.set_ylim(self.ymin, self.ymax)
309 if self.xaxis == 'time':
310 if self.xaxis == 'time':
310 ax.xaxis.set_major_formatter(FuncFormatter(func))
311 ax.xaxis.set_major_formatter(FuncFormatter(func))
311 ax.xaxis.set_major_locator(LinearLocator(6))
312 ax.xaxis.set_major_locator(LinearLocator(6))
312
313
313 # ax.yaxis.set_major_locator(LinearLocator(4))
314 # ax.yaxis.set_major_locator(LinearLocator(4))
314
315
315 ax.set_ylabel(self.ylabel)
316 ax.set_ylabel(self.ylabel)
316
317
317 # if self.xmin is None:
318 # if self.xmin is None:
318 # xmin = self.min_time
319 # xmin = self.min_time
319 # else:
320 # else:
320 # xmin = (datetime.datetime.combine(self.dataOut.datatime.date(),
321 # xmin = (datetime.datetime.combine(self.dataOut.datatime.date(),
321 # datetime.time(self.xmin, 0, 0))-d1970).total_seconds()
322 # datetime.time(self.xmin, 0, 0))-d1970).total_seconds()
322
323
323 ax.set_xlim(xmin, xmax)
324 ax.set_xlim(xmin, xmax)
324 ax.firsttime = False
325 ax.firsttime = False
325 else:
326 else:
326 ax.collections.remove(ax.collections[0])
327 ax.collections.remove(ax.collections[0])
327 ax.set_xlim(xmin, xmax)
328 ax.set_xlim(xmin, xmax)
328 plot = ax.pcolormesh(x, y, z[n].T,
329 plot = ax.pcolormesh(x, y, z[n].T,
329 vmin=self.zmin,
330 vmin=self.zmin,
330 vmax=self.zmax,
331 vmax=self.zmax,
331 cmap=plt.get_cmap(self.colormap)
332 cmap=plt.get_cmap(self.colormap)
332 )
333 )
333 ax.set_title('{} {}'.format(self.titles[n],
334 ax.set_title('{} {}'.format(self.titles[n],
334 datetime.datetime.utcfromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
335 datetime.datetime.utcfromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
335 size=8)
336 size=8)
336
337
337
338
338 class PlotCOHData(PlotRTIData):
339 class PlotCOHData(PlotRTIData):
339
340
340 CODE = 'coh'
341 CODE = 'coh'
341
342
342 def setup(self):
343 def setup(self):
343
344
344 self.ncols = 1
345 self.ncols = 1
345 self.nrows = self.dataOut.nPairs
346 self.nrows = self.dataOut.nPairs
346 self.width = 10
347 self.width = 10
347 self.height = 2.2*self.nrows
348 self.height = 2.2*self.nrows
348 self.ylabel = 'Range [Km]'
349 self.ylabel = 'Range [Km]'
349 self.titles = ['Channels {}'.format(x) for x in self.dataOut.pairsList]
350 self.titles = ['Channels {}'.format(x) for x in self.dataOut.pairsList]
350
351
351 if self.figure is None:
352 if self.figure is None:
352 self.figure = plt.figure(figsize=(self.width, self.height),
353 self.figure = plt.figure(figsize=(self.width, self.height),
353 edgecolor='k',
354 edgecolor='k',
354 facecolor='w')
355 facecolor='w')
355 else:
356 else:
356 self.figure.clf()
357 self.figure.clf()
357
358
358 for n in range(self.nrows):
359 for n in range(self.nrows):
359 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
360 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
360 ax.firsttime = True
361 ax.firsttime = True
361 self.axes.append(ax)
362 self.axes.append(ax)
362
363
363 self.figure.subplots_adjust(hspace=0.5)
364 self.figure.subplots_adjust(hspace=0.5)
364 self.figure.show()
365 self.figure.show()
365
366
366 class PlotSNRData(PlotRTIData):
367 class PlotSNRData(PlotRTIData):
367
368
368 CODE = 'snr'
369 CODE = 'snr'
369
370
370 class PlotDOPData(PlotRTIData):
371 class PlotDOPData(PlotRTIData):
371 CODE = 'dop'
372 CODE = 'dop'
372 colormap = 'jet'
373 colormap = 'jet'
373
374
374 class PlotPHASEData(PlotCOHData):
375 class PlotPHASEData(PlotCOHData):
375
376
376 CODE = 'phase'
377 CODE = 'phase'
377 colormap = 'seismic'
378 colormap = 'seismic'
Show file before | Show file after | Hide comments
@@ -1,299 +1,315
1 '''
1 '''
2
2
3 $Author: murco $
3 $Author: murco $
4 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
4 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
5 '''
5 '''
6
6
7 class ProcessingUnit(object):
7 class ProcessingUnit(object):
8
8
9 """
9 """
10 Esta es la clase base para el procesamiento de datos.
10 Esta es la clase base para el procesamiento de datos.
11
11
12 Contiene el metodo "call" para llamar operaciones. Las operaciones pueden ser:
12 Contiene el metodo "call" para llamar operaciones. Las operaciones pueden ser:
13 - Metodos internos (callMethod)
13 - Metodos internos (callMethod)
14 - Objetos del tipo Operation (callObject). Antes de ser llamados, estos objetos
14 - Objetos del tipo Operation (callObject). Antes de ser llamados, estos objetos
15 tienen que ser agreagados con el metodo "add".
15 tienen que ser agreagados con el metodo "add".
16
16
17 """
17 """
18 # objeto de datos de entrada (Voltage, Spectra o Correlation)
18 # objeto de datos de entrada (Voltage, Spectra o Correlation)
19 dataIn = None
19 dataIn = None
20 dataInList = []
20 dataInList = []
21
21
22 # objeto de datos de entrada (Voltage, Spectra o Correlation)
22 # objeto de datos de entrada (Voltage, Spectra o Correlation)
23 dataOut = None
23 dataOut = None
24
24
25 operations2RunDict = None
25 operations2RunDict = None
26
26
27 isConfig = False
27 isConfig = False
28
28
29
29
30 def __init__(self, *args, **kwargs):
30 def __init__(self, *args, **kwargs):
31
31
32 self.dataIn = None
32 self.dataIn = None
33 self.dataInList = []
33 self.dataInList = []
34
34
35 self.dataOut = None
35 self.dataOut = None
36
36
37 self.operations2RunDict = {}
37 self.operations2RunDict = {}
38 self.operationKwargs = {}
38
39
39 self.isConfig = False
40 self.isConfig = False
40
41
41 self.args = args
42 self.args = args
42 self.kwargs = kwargs
43 self.kwargs = kwargs
43
44
45 def addOperationKwargs(self, objId, **kwargs):
46 '''
47 '''
48
49 self.operationKwargs[objId] = kwargs
50
51
44 def addOperation(self, opObj, objId):
52 def addOperation(self, opObj, objId):
45
53
46 """
54 """
47 Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
55 Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
48 identificador asociado a este objeto.
56 identificador asociado a este objeto.
49
57
50 Input:
58 Input:
51
59
52 object : objeto de la clase "Operation"
60 object : objeto de la clase "Operation"
53
61
54 Return:
62 Return:
55
63
56 objId : identificador del objeto, necesario para ejecutar la operacion
64 objId : identificador del objeto, necesario para ejecutar la operacion
57 """
65 """
58
66
59 self.operations2RunDict[objId] = opObj
67 self.operations2RunDict[objId] = opObj
60
68
61 return objId
69 return objId
62
70
63 def getOperationObj(self, objId):
71 def getOperationObj(self, objId):
64
72
65 if objId not in self.operations2RunDict.keys():
73 if objId not in self.operations2RunDict.keys():
66 return None
74 return None
67
75
68 return self.operations2RunDict[objId]
76 return self.operations2RunDict[objId]
69
77
70 def operation(self, **kwargs):
78 def operation(self, **kwargs):
71
79
72 """
80 """
73 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
81 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
74 atributos del objeto dataOut
82 atributos del objeto dataOut
75
83
76 Input:
84 Input:
77
85
78 **kwargs : Diccionario de argumentos de la funcion a ejecutar
86 **kwargs : Diccionario de argumentos de la funcion a ejecutar
79 """
87 """
80
88
81 raise NotImplementedError
89 raise NotImplementedError
82
90
83 def callMethod(self, name, **kwargs):
91 def callMethod(self, name, opId):
84
92
85 """
93 """
86 Ejecuta el metodo con el nombre "name" y con argumentos **kwargs de la propia clase.
94 Ejecuta el metodo con el nombre "name" y con argumentos **kwargs de la propia clase.
87
95
88 Input:
96 Input:
89 name : nombre del metodo a ejecutar
97 name : nombre del metodo a ejecutar
90
98
91 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
99 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
92
100
93 """
101 """
94
102
95 #Checking the inputs
103 #Checking the inputs
96 if name == 'run':
104 if name == 'run':
97
105
98 if not self.checkInputs():
106 if not self.checkInputs():
99 self.dataOut.flagNoData = True
107 self.dataOut.flagNoData = True
100 return False
108 return False
101 else:
109 else:
102 #Si no es un metodo RUN la entrada es la misma dataOut (interna)
110 #Si no es un metodo RUN la entrada es la misma dataOut (interna)
103 if self.dataOut.isEmpty():
111 if self.dataOut is not None and self.dataOut.isEmpty():
104 return False
112 return False
105
113
106 #Getting the pointer to method
114 #Getting the pointer to method
107 methodToCall = getattr(self, name)
115 methodToCall = getattr(self, name)
108
116
109 #Executing the self method
117 #Executing the self method
110
118
111 if hasattr(self, 'mp'):
119 if hasattr(self, 'mp'):
112 if self.mp is False:
120 if name=='run':
113 self.mp = True
121 if self.mp is False:
114 self.start()
122 self.mp = True
123 self.start()
124 else:
125 methodToCall(**self.operationKwargs[opId])
115 else:
126 else:
116 methodToCall(**kwargs)
127 if name=='run':
128 methodToCall(**self.kwargs)
129 else:
130 methodToCall(**self.operationKwargs[opId])
117
131
118 if self.dataOut is None:
132 if self.dataOut is None:
119 return False
133 return False
120
134
121 if self.dataOut.isEmpty():
135 if self.dataOut.isEmpty():
122 return False
136 return False
123
137
124 return True
138 return True
125
139
126 def callObject(self, objId):
140 def callObject(self, objId):
127
141
128 """
142 """
129 Ejecuta la operacion asociada al identificador del objeto "objId"
143 Ejecuta la operacion asociada al identificador del objeto "objId"
130
144
131 Input:
145 Input:
132
146
133 objId : identificador del objeto a ejecutar
147 objId : identificador del objeto a ejecutar
134
148
135 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
149 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
136
150
137 Return:
151 Return:
138
152
139 None
153 None
140 """
154 """
141
155
142 if self.dataOut is not None and self.dataOut.isEmpty():
156 if self.dataOut is not None and self.dataOut.isEmpty():
143 return False
157 return False
144
158
145 externalProcObj = self.operations2RunDict[objId]
159 externalProcObj = self.operations2RunDict[objId]
146
160
147 if hasattr(externalProcObj, 'mp'):
161 if hasattr(externalProcObj, 'mp'):
148 if externalProcObj.mp is False:
162 if externalProcObj.mp is False:
163 self.operationKwargs[objId] = externalProcObj.kwargs
149 externalProcObj.mp = True
164 externalProcObj.mp = True
150 externalProcObj.start()
165 externalProcObj.start()
151 else:
166 else:
152 externalProcObj.run(self.dataOut, **externalProcObj.kwargs)
167 externalProcObj.run(self.dataOut, **externalProcObj.kwargs)
168 self.operationKwargs[objId] = externalProcObj.kwargs
153
169
154 return True
170 return True
155
171
156 def call(self, opType, opName=None, opId=None):
172 def call(self, opType, opName=None, opId=None):
157
173
158 """
174 """
159 Return True si ejecuta la operacion interna nombrada "opName" o la operacion externa
175 Return True si ejecuta la operacion interna nombrada "opName" o la operacion externa
160 identificada con el id "opId"; con los argumentos "**kwargs".
176 identificada con el id "opId"; con los argumentos "**kwargs".
161
177
162 False si la operacion no se ha ejecutado.
178 False si la operacion no se ha ejecutado.
163
179
164 Input:
180 Input:
165
181
166 opType : Puede ser "self" o "external"
182 opType : Puede ser "self" o "external"
167
183
168 Depende del tipo de operacion para llamar a:callMethod or callObject:
184 Depende del tipo de operacion para llamar a:callMethod or callObject:
169
185
170 1. If opType = "self": Llama a un metodo propio de esta clase:
186 1. If opType = "self": Llama a un metodo propio de esta clase:
171
187
172 name_method = getattr(self, name)
188 name_method = getattr(self, name)
173 name_method(**kwargs)
189 name_method(**kwargs)
174
190
175
191
176 2. If opType = "other" o"external": Llama al metodo "run()" de una instancia de la
192 2. If opType = "other" o"external": Llama al metodo "run()" de una instancia de la
177 clase "Operation" o de un derivado de ella:
193 clase "Operation" o de un derivado de ella:
178
194
179 instanceName = self.operationList[opId]
195 instanceName = self.operationList[opId]
180 instanceName.run(**kwargs)
196 instanceName.run(**kwargs)
181
197
182 opName : Si la operacion es interna (opType = 'self'), entonces el "opName" sera
198 opName : Si la operacion es interna (opType = 'self'), entonces el "opName" sera
183 usada para llamar a un metodo interno de la clase Processing
199 usada para llamar a un metodo interno de la clase Processing
184
200
185 opId : Si la operacion es externa (opType = 'other' o 'external), entonces el
201 opId : Si la operacion es externa (opType = 'other' o 'external), entonces el
186 "opId" sera usada para llamar al metodo "run" de la clase Operation
202 "opId" sera usada para llamar al metodo "run" de la clase Operation
187 registrada anteriormente con ese Id
203 registrada anteriormente con ese Id
188
204
189 Exception:
205 Exception:
190 Este objeto de tipo Operation debe de haber sido agregado antes con el metodo:
206 Este objeto de tipo Operation debe de haber sido agregado antes con el metodo:
191 "addOperation" e identificado con el valor "opId" = el id de la operacion.
207 "addOperation" e identificado con el valor "opId" = el id de la operacion.
192 De lo contrario retornara un error del tipo ValueError
208 De lo contrario retornara un error del tipo ValueError
193
209
194 """
210 """
195
211
196 if opType == 'self':
212 if opType == 'self':
197
213
198 if not opName:
214 if not opName:
199 raise ValueError, "opName parameter should be defined"
215 raise ValueError, "opName parameter should be defined"
200
216
201 sts = self.callMethod(opName, **self.kwargs)
217 sts = self.callMethod(opName, opId)
202
218
203 elif opType == 'other' or opType == 'external' or opType == 'plotter':
219 elif opType == 'other' or opType == 'external' or opType == 'plotter':
204
220
205 if not opId:
221 if not opId:
206 raise ValueError, "opId parameter should be defined"
222 raise ValueError, "opId parameter should be defined"
207
223
208 if opId not in self.operations2RunDict.keys():
224 if opId not in self.operations2RunDict.keys():
209 raise ValueError, "Any operation with id=%s has been added" %str(opId)
225 raise ValueError, "Any operation with id=%s has been added" %str(opId)
210
226
211 sts = self.callObject(opId)
227 sts = self.callObject(opId)
212
228
213 else:
229 else:
214 raise ValueError, "opType should be 'self', 'external' or 'plotter'; and not '%s'" %opType
230 raise ValueError, "opType should be 'self', 'external' or 'plotter'; and not '%s'" %opType
215
231
216 return sts
232 return sts
217
233
218 def setInput(self, dataIn):
234 def setInput(self, dataIn):
219
235
220 self.dataIn = dataIn
236 self.dataIn = dataIn
221 self.dataInList.append(dataIn)
237 self.dataInList.append(dataIn)
222
238
223 def getOutputObj(self):
239 def getOutputObj(self):
224
240
225 return self.dataOut
241 return self.dataOut
226
242
227 def checkInputs(self):
243 def checkInputs(self):
228
244
229 for thisDataIn in self.dataInList:
245 for thisDataIn in self.dataInList:
230
246
231 if thisDataIn.isEmpty():
247 if thisDataIn.isEmpty():
232 return False
248 return False
233
249
234 return True
250 return True
235
251
236 def setup(self):
252 def setup(self):
237
253
238 raise NotImplementedError
254 raise NotImplementedError
239
255
240 def run(self):
256 def run(self):
241
257
242 raise NotImplementedError
258 raise NotImplementedError
243
259
244 def close(self):
260 def close(self):
245 #Close every thread, queue or any other object here is it is neccesary.
261 #Close every thread, queue or any other object here is it is neccesary.
246 return
262 return
247
263
248 class Operation(object):
264 class Operation(object):
249
265
250 """
266 """
251 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
267 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
252 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
268 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
253 acumulacion dentro de esta clase
269 acumulacion dentro de esta clase
254
270
255 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
271 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
256
272
257 """
273 """
258
274
259 __buffer = None
275 __buffer = None
260 isConfig = False
276 isConfig = False
261
277
262 def __init__(self, **kwargs):
278 def __init__(self, **kwargs):
263
279
264 self.__buffer = None
280 self.__buffer = None
265 self.isConfig = False
281 self.isConfig = False
266 self.kwargs = kwargs
282 self.kwargs = kwargs
267
283
268 def setup(self):
284 def setup(self):
269
285
270 self.isConfig = True
286 self.isConfig = True
271
287
272 raise NotImplementedError
288 raise NotImplementedError
273
289
274 def run(self, dataIn, **kwargs):
290 def run(self, dataIn, **kwargs):
275
291
276 """
292 """
277 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
293 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
278 atributos del objeto dataIn.
294 atributos del objeto dataIn.
279
295
280 Input:
296 Input:
281
297
282 dataIn : objeto del tipo JROData
298 dataIn : objeto del tipo JROData
283
299
284 Return:
300 Return:
285
301
286 None
302 None
287
303
288 Affected:
304 Affected:
289 __buffer : buffer de recepcion de datos.
305 __buffer : buffer de recepcion de datos.
290
306
291 """
307 """
292 if not self.isConfig:
308 if not self.isConfig:
293 self.setup(**kwargs)
309 self.setup(**kwargs)
294
310
295 raise NotImplementedError
311 raise NotImplementedError
296
312
297 def close(self):
313 def close(self):
298
314
299 pass
315 pass
Show file before | Show file after | Hide comments
@@ -1,2746 +1,2741
1 import numpy
1 import numpy
2 import math
2 import math
3 from scipy import optimize, interpolate, signal, stats, ndimage
3 from scipy import optimize, interpolate, signal, stats, ndimage
4 import re
4 import re
5 import datetime
5 import datetime
6 import copy
6 import copy
7 import sys
7 import sys
8 import importlib
8 import importlib
9 import itertools
9 import itertools
10
10
11 from jroproc_base import ProcessingUnit, Operation
11 from jroproc_base import ProcessingUnit, Operation
12 from schainpy.model.data.jrodata import Parameters, hildebrand_sekhon
12 from schainpy.model.data.jrodata import Parameters, hildebrand_sekhon
13
13
14
14
15 class ParametersProc(ProcessingUnit):
15 class ParametersProc(ProcessingUnit):
16
16
17 nSeconds = None
17 nSeconds = None
18
18
19 def __init__(self):
19 def __init__(self):
20 ProcessingUnit.__init__(self)
20 ProcessingUnit.__init__(self)
21
21
22 # self.objectDict = {}
22 # self.objectDict = {}
23 self.buffer = None
23 self.buffer = None
24 self.firstdatatime = None
24 self.firstdatatime = None
25 self.profIndex = 0
25 self.profIndex = 0
26 self.dataOut = Parameters()
26 self.dataOut = Parameters()
27
27
28 def __updateObjFromInput(self):
28 def __updateObjFromInput(self):
29
29
30 self.dataOut.inputUnit = self.dataIn.type
30 self.dataOut.inputUnit = self.dataIn.type
31
31
32 self.dataOut.timeZone = self.dataIn.timeZone
32 self.dataOut.timeZone = self.dataIn.timeZone
33 self.dataOut.dstFlag = self.dataIn.dstFlag
33 self.dataOut.dstFlag = self.dataIn.dstFlag
34 self.dataOut.errorCount = self.dataIn.errorCount
34 self.dataOut.errorCount = self.dataIn.errorCount
35 self.dataOut.useLocalTime = self.dataIn.useLocalTime
35 self.dataOut.useLocalTime = self.dataIn.useLocalTime
36
36
37 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
37 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
38 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
38 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
39 self.dataOut.channelList = self.dataIn.channelList
39 self.dataOut.channelList = self.dataIn.channelList
40 self.dataOut.heightList = self.dataIn.heightList
40 self.dataOut.heightList = self.dataIn.heightList
41 self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
41 self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
42 # self.dataOut.nHeights = self.dataIn.nHeights
42 # self.dataOut.nHeights = self.dataIn.nHeights
43 # self.dataOut.nChannels = self.dataIn.nChannels
43 # self.dataOut.nChannels = self.dataIn.nChannels
44 self.dataOut.nBaud = self.dataIn.nBaud
44 self.dataOut.nBaud = self.dataIn.nBaud
45 self.dataOut.nCode = self.dataIn.nCode
45 self.dataOut.nCode = self.dataIn.nCode
46 self.dataOut.code = self.dataIn.code
46 self.dataOut.code = self.dataIn.code
47 # self.dataOut.nProfiles = self.dataOut.nFFTPoints
47 # self.dataOut.nProfiles = self.dataOut.nFFTPoints
48 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
48 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
49 # self.dataOut.utctime = self.firstdatatime
49 # self.dataOut.utctime = self.firstdatatime
50 self.dataOut.utctime = self.dataIn.utctime
50 self.dataOut.utctime = self.dataIn.utctime
51 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
51 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
52 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip
52 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip
53 self.dataOut.nCohInt = self.dataIn.nCohInt
53 self.dataOut.nCohInt = self.dataIn.nCohInt
54 # self.dataOut.nIncohInt = 1
54 # self.dataOut.nIncohInt = 1
55 self.dataOut.ippSeconds = self.dataIn.ippSeconds
55 self.dataOut.ippSeconds = self.dataIn.ippSeconds
56 # self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
56 # self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
57 self.dataOut.timeInterval = self.dataIn.timeInterval
57 self.dataOut.timeInterval = self.dataIn.timeInterval
58 self.dataOut.heightList = self.dataIn.getHeiRange()
58 self.dataOut.heightList = self.dataIn.getHeiRange()
59 self.dataOut.frequency = self.dataIn.frequency
59 self.dataOut.frequency = self.dataIn.frequency
60 self.dataOut.noise = self.dataIn.noise
60 self.dataOut.noise = self.dataIn.noise
61
61
62 def run(self):
62 def run(self):
63
63
64 #---------------------- Voltage Data ---------------------------
64 #---------------------- Voltage Data ---------------------------
65
65
66 if self.dataIn.type == "Voltage":
66 if self.dataIn.type == "Voltage":
67
67
68 self.__updateObjFromInput()
68 self.__updateObjFromInput()
69 self.dataOut.data_pre = self.dataIn.data.copy()
69 self.dataOut.data_pre = self.dataIn.data.copy()
70 self.dataOut.flagNoData = False
70 self.dataOut.flagNoData = False
71 self.dataOut.utctimeInit = self.dataIn.utctime
71 self.dataOut.utctimeInit = self.dataIn.utctime
72 self.dataOut.paramInterval = self.dataIn.nProfiles*self.dataIn.nCohInt*self.dataIn.ippSeconds
72 self.dataOut.paramInterval = self.dataIn.nProfiles*self.dataIn.nCohInt*self.dataIn.ippSeconds
73 return
73 return
74
74
75 #---------------------- Spectra Data ---------------------------
75 #---------------------- Spectra Data ---------------------------
76
76
77 if self.dataIn.type == "Spectra":
77 if self.dataIn.type == "Spectra":
78
78
79 self.dataOut.data_pre = (self.dataIn.data_spc,self.dataIn.data_cspc)
79 self.dataOut.data_pre = (self.dataIn.data_spc,self.dataIn.data_cspc)
80 self.dataOut.abscissaList = self.dataIn.getVelRange(1)
80 self.dataOut.abscissaList = self.dataIn.getVelRange(1)
81 # self.dataOut.noise = self.dataIn.getNoise()
81 # self.dataOut.noise = self.dataIn.getNoise()
82 self.dataOut.normFactor = self.dataIn.normFactor
82 self.dataOut.normFactor = self.dataIn.normFactor
83 self.dataOut.groupList = self.dataIn.pairsList
83 self.dataOut.groupList = self.dataIn.pairsList
84 self.dataOut.flagNoData = False
84 self.dataOut.flagNoData = False
85
85
86 #---------------------- Correlation Data ---------------------------
86 #---------------------- Correlation Data ---------------------------
87
87
88 if self.dataIn.type == "Correlation":
88 if self.dataIn.type == "Correlation":
89 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.dataIn.splitFunctions()
89 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.dataIn.splitFunctions()
90
90
91 self.dataOut.data_pre = (self.dataIn.data_cf[acf_ind,:], self.dataIn.data_cf[ccf_ind,:,:])
91 self.dataOut.data_pre = (self.dataIn.data_cf[acf_ind,:], self.dataIn.data_cf[ccf_ind,:,:])
92 self.dataOut.normFactor = (self.dataIn.normFactor[acf_ind,:], self.dataIn.normFactor[ccf_ind,:])
92 self.dataOut.normFactor = (self.dataIn.normFactor[acf_ind,:], self.dataIn.normFactor[ccf_ind,:])
93 self.dataOut.groupList = (acf_pairs, ccf_pairs)
93 self.dataOut.groupList = (acf_pairs, ccf_pairs)
94
94
95 self.dataOut.abscissaList = self.dataIn.lagRange
95 self.dataOut.abscissaList = self.dataIn.lagRange
96 self.dataOut.noise = self.dataIn.noise
96 self.dataOut.noise = self.dataIn.noise
97 self.dataOut.data_SNR = self.dataIn.SNR
97 self.dataOut.data_SNR = self.dataIn.SNR
98 self.dataOut.flagNoData = False
98 self.dataOut.flagNoData = False
99 self.dataOut.nAvg = self.dataIn.nAvg
99 self.dataOut.nAvg = self.dataIn.nAvg
100
100
101 #---------------------- Parameters Data ---------------------------
101 #---------------------- Parameters Data ---------------------------
102
102
103 if self.dataIn.type == "Parameters":
103 if self.dataIn.type == "Parameters":
104 self.dataOut.copy(self.dataIn)
104 self.dataOut.copy(self.dataIn)
105 self.dataOut.utctimeInit = self.dataIn.utctime
105 self.dataOut.utctimeInit = self.dataIn.utctime
106 self.dataOut.flagNoData = False
106 self.dataOut.flagNoData = False
107
107
108 return True
108 return True
109
109
110 self.__updateObjFromInput()
110 self.__updateObjFromInput()
111 self.dataOut.utctimeInit = self.dataIn.utctime
111 self.dataOut.utctimeInit = self.dataIn.utctime
112 self.dataOut.paramInterval = self.dataIn.timeInterval
112 self.dataOut.paramInterval = self.dataIn.timeInterval
113
113
114 return
114 return
115
115
116 class SpectralMoments(Operation):
116 class SpectralMoments(Operation):
117
117
118 '''
118 '''
119 Function SpectralMoments()
119 Function SpectralMoments()
120
120
121 Calculates moments (power, mean, standard deviation) and SNR of the signal
121 Calculates moments (power, mean, standard deviation) and SNR of the signal
122
122
123 Type of dataIn: Spectra
123 Type of dataIn: Spectra
124
124
125 Configuration Parameters:
125 Configuration Parameters:
126
126
127 dirCosx : Cosine director in X axis
127 dirCosx : Cosine director in X axis
128 dirCosy : Cosine director in Y axis
128 dirCosy : Cosine director in Y axis
129
129
130 elevation :
130 elevation :
131 azimuth :
131 azimuth :
132
132
133 Input:
133 Input:
134 channelList : simple channel list to select e.g. [2,3,7]
134 channelList : simple channel list to select e.g. [2,3,7]
135 self.dataOut.data_pre : Spectral data
135 self.dataOut.data_pre : Spectral data
136 self.dataOut.abscissaList : List of frequencies
136 self.dataOut.abscissaList : List of frequencies
137 self.dataOut.noise : Noise level per channel
137 self.dataOut.noise : Noise level per channel
138
138
139 Affected:
139 Affected:
140 self.dataOut.data_param : Parameters per channel
140 self.dataOut.data_param : Parameters per channel
141 self.dataOut.data_SNR : SNR per channel
141 self.dataOut.data_SNR : SNR per channel
142
142
143 '''
143 '''
144
144
145 def run(self, dataOut):
145 def run(self, dataOut):
146
146
147 dataOut.data_pre = dataOut.data_pre[0]
147 dataOut.data_pre = dataOut.data_pre[0]
148 data = dataOut.data_pre
148 data = dataOut.data_pre
149 absc = dataOut.abscissaList[:-1]
149 absc = dataOut.abscissaList[:-1]
150 noise = dataOut.noise
150 noise = dataOut.noise
151 nChannel = data.shape[0]
151 nChannel = data.shape[0]
152 data_param = numpy.zeros((nChannel, 4, data.shape[2]))
152 data_param = numpy.zeros((nChannel, 4, data.shape[2]))
153
153
154 for ind in range(nChannel):
154 for ind in range(nChannel):
155 data_param[ind,:,:] = self.__calculateMoments(data[ind,:,:], absc, noise[ind])
155 data_param[ind,:,:] = self.__calculateMoments(data[ind,:,:], absc, noise[ind])
156
156
157 dataOut.data_param = data_param[:,1:,:]
157 dataOut.data_param = data_param[:,1:,:]
158 dataOut.data_SNR = data_param[:,0]
158 dataOut.data_SNR = data_param[:,0]
159 dataOut.data_DOP = data_param[:,1]
159 dataOut.data_DOP = data_param[:,1]
160 return
160 return
161
161
162 def __calculateMoments(self, oldspec, oldfreq, n0, nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None):
162 def __calculateMoments(self, oldspec, oldfreq, n0, nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None):
163
163
164 if (nicoh is None): nicoh = 1
164 if (nicoh is None): nicoh = 1
165 if (graph is None): graph = 0
165 if (graph is None): graph = 0
166 if (smooth is None): smooth = 0
166 if (smooth is None): smooth = 0
167 elif (self.smooth < 3): smooth = 0
167 elif (self.smooth < 3): smooth = 0
168
168
169 if (type1 is None): type1 = 0
169 if (type1 is None): type1 = 0
170 if (fwindow is None): fwindow = numpy.zeros(oldfreq.size) + 1
170 if (fwindow is None): fwindow = numpy.zeros(oldfreq.size) + 1
171 if (snrth is None): snrth = -3
171 if (snrth is None): snrth = -3
172 if (dc is None): dc = 0
172 if (dc is None): dc = 0
173 if (aliasing is None): aliasing = 0
173 if (aliasing is None): aliasing = 0
174 if (oldfd is None): oldfd = 0
174 if (oldfd is None): oldfd = 0
175 if (wwauto is None): wwauto = 0
175 if (wwauto is None): wwauto = 0
176
176
177 if (n0 < 1.e-20): n0 = 1.e-20
177 if (n0 < 1.e-20): n0 = 1.e-20
178
178
179 freq = oldfreq
179 freq = oldfreq
180 vec_power = numpy.zeros(oldspec.shape[1])
180 vec_power = numpy.zeros(oldspec.shape[1])
181 vec_fd = numpy.zeros(oldspec.shape[1])
181 vec_fd = numpy.zeros(oldspec.shape[1])
182 vec_w = numpy.zeros(oldspec.shape[1])
182 vec_w = numpy.zeros(oldspec.shape[1])
183 vec_snr = numpy.zeros(oldspec.shape[1])
183 vec_snr = numpy.zeros(oldspec.shape[1])
184
184
185 for ind in range(oldspec.shape[1]):
185 for ind in range(oldspec.shape[1]):
186
186
187 spec = oldspec[:,ind]
187 spec = oldspec[:,ind]
188 aux = spec*fwindow
188 aux = spec*fwindow
189 max_spec = aux.max()
189 max_spec = aux.max()
190 m = list(aux).index(max_spec)
190 m = list(aux).index(max_spec)
191
191
192 #Smooth
192 #Smooth
193 if (smooth == 0): spec2 = spec
193 if (smooth == 0): spec2 = spec
194 else: spec2 = scipy.ndimage.filters.uniform_filter1d(spec,size=smooth)
194 else: spec2 = scipy.ndimage.filters.uniform_filter1d(spec,size=smooth)
195
195
196 # Calculo de Momentos
196 # Calculo de Momentos
197 bb = spec2[range(m,spec2.size)]
197 bb = spec2[range(m,spec2.size)]
198 bb = (bb<n0).nonzero()
198 bb = (bb<n0).nonzero()
199 bb = bb[0]
199 bb = bb[0]
200
200
201 ss = spec2[range(0,m + 1)]
201 ss = spec2[range(0,m + 1)]
202 ss = (ss<n0).nonzero()
202 ss = (ss<n0).nonzero()
203 ss = ss[0]
203 ss = ss[0]
204
204
205 if (bb.size == 0):
205 if (bb.size == 0):
206 bb0 = spec.size - 1 - m
206 bb0 = spec.size - 1 - m
207 else:
207 else:
208 bb0 = bb[0] - 1
208 bb0 = bb[0] - 1
209 if (bb0 < 0):
209 if (bb0 < 0):
210 bb0 = 0
210 bb0 = 0
211
211
212 if (ss.size == 0): ss1 = 1
212 if (ss.size == 0): ss1 = 1
213 else: ss1 = max(ss) + 1
213 else: ss1 = max(ss) + 1
214
214
215 if (ss1 > m): ss1 = m
215 if (ss1 > m): ss1 = m
216
216
217 valid = numpy.asarray(range(int(m + bb0 - ss1 + 1))) + ss1
217 valid = numpy.asarray(range(int(m + bb0 - ss1 + 1))) + ss1
218 power = ((spec2[valid] - n0)*fwindow[valid]).sum()
218 power = ((spec2[valid] - n0)*fwindow[valid]).sum()
219 fd = ((spec2[valid]- n0)*freq[valid]*fwindow[valid]).sum()/power
219 fd = ((spec2[valid]- n0)*freq[valid]*fwindow[valid]).sum()/power
220 w = math.sqrt(((spec2[valid] - n0)*fwindow[valid]*(freq[valid]- fd)**2).sum()/power)
220 w = math.sqrt(((spec2[valid] - n0)*fwindow[valid]*(freq[valid]- fd)**2).sum()/power)
221 snr = (spec2.mean()-n0)/n0
221 snr = (spec2.mean()-n0)/n0
222
222
223 if (snr < 1.e-20) :
223 if (snr < 1.e-20) :
224 snr = 1.e-20
224 snr = 1.e-20
225
225
226 vec_power[ind] = power
226 vec_power[ind] = power
227 vec_fd[ind] = fd
227 vec_fd[ind] = fd
228 vec_w[ind] = w
228 vec_w[ind] = w
229 vec_snr[ind] = snr
229 vec_snr[ind] = snr
230
230
231 moments = numpy.vstack((vec_snr, vec_power, vec_fd, vec_w))
231 moments = numpy.vstack((vec_snr, vec_power, vec_fd, vec_w))
232 return moments
232 return moments
233
233
234 #------------------ Get SA Parameters --------------------------
234 #------------------ Get SA Parameters --------------------------
235
235
236 def GetSAParameters(self):
236 def GetSAParameters(self):
237 #SA en frecuencia
237 #SA en frecuencia
238 pairslist = self.dataOut.groupList
238 pairslist = self.dataOut.groupList
239 num_pairs = len(pairslist)
239 num_pairs = len(pairslist)
240
240
241 vel = self.dataOut.abscissaList
241 vel = self.dataOut.abscissaList
242 spectra = self.dataOut.data_pre
242 spectra = self.dataOut.data_pre
243 cspectra = self.dataIn.data_cspc
243 cspectra = self.dataIn.data_cspc
244 delta_v = vel[1] - vel[0]
244 delta_v = vel[1] - vel[0]
245
245
246 #Calculating the power spectrum
246 #Calculating the power spectrum
247 spc_pow = numpy.sum(spectra, 3)*delta_v
247 spc_pow = numpy.sum(spectra, 3)*delta_v
248 #Normalizing Spectra
248 #Normalizing Spectra
249 norm_spectra = spectra/spc_pow
249 norm_spectra = spectra/spc_pow
250 #Calculating the norm_spectra at peak
250 #Calculating the norm_spectra at peak
251 max_spectra = numpy.max(norm_spectra, 3)
251 max_spectra = numpy.max(norm_spectra, 3)
252
252
253 #Normalizing Cross Spectra
253 #Normalizing Cross Spectra
254 norm_cspectra = numpy.zeros(cspectra.shape)
254 norm_cspectra = numpy.zeros(cspectra.shape)
255
255
256 for i in range(num_chan):
256 for i in range(num_chan):
257 norm_cspectra[i,:,:] = cspectra[i,:,:]/numpy.sqrt(spc_pow[pairslist[i][0],:]*spc_pow[pairslist[i][1],:])
257 norm_cspectra[i,:,:] = cspectra[i,:,:]/numpy.sqrt(spc_pow[pairslist[i][0],:]*spc_pow[pairslist[i][1],:])
258
258
259 max_cspectra = numpy.max(norm_cspectra,2)
259 max_cspectra = numpy.max(norm_cspectra,2)
260 max_cspectra_index = numpy.argmax(norm_cspectra, 2)
260 max_cspectra_index = numpy.argmax(norm_cspectra, 2)
261
261
262 for i in range(num_pairs):
262 for i in range(num_pairs):
263 cspc_par[i,:,:] = __calculateMoments(norm_cspectra)
263 cspc_par[i,:,:] = __calculateMoments(norm_cspectra)
264 #------------------- Get Lags ----------------------------------
264 #------------------- Get Lags ----------------------------------
265
265
266 class SALags(Operation):
266 class SALags(Operation):
267 '''
267 '''
268 Function GetMoments()
268 Function GetMoments()
269
269
270 Input:
270 Input:
271 self.dataOut.data_pre
271 self.dataOut.data_pre
272 self.dataOut.abscissaList
272 self.dataOut.abscissaList
273 self.dataOut.noise
273 self.dataOut.noise
274 self.dataOut.normFactor
274 self.dataOut.normFactor
275 self.dataOut.data_SNR
275 self.dataOut.data_SNR
276 self.dataOut.groupList
276 self.dataOut.groupList
277 self.dataOut.nChannels
277 self.dataOut.nChannels
278
278
279 Affected:
279 Affected:
280 self.dataOut.data_param
280 self.dataOut.data_param
281
281
282 '''
282 '''
283 def run(self, dataOut):
283 def run(self, dataOut):
284 data_acf = dataOut.data_pre[0]
284 data_acf = dataOut.data_pre[0]
285 data_ccf = dataOut.data_pre[1]
285 data_ccf = dataOut.data_pre[1]
286 normFactor_acf = dataOut.normFactor[0]
286 normFactor_acf = dataOut.normFactor[0]
287 normFactor_ccf = dataOut.normFactor[1]
287 normFactor_ccf = dataOut.normFactor[1]
288 pairs_acf = dataOut.groupList[0]
288 pairs_acf = dataOut.groupList[0]
289 pairs_ccf = dataOut.groupList[1]
289 pairs_ccf = dataOut.groupList[1]
290
290
291 nHeights = dataOut.nHeights
291 nHeights = dataOut.nHeights
292 absc = dataOut.abscissaList
292 absc = dataOut.abscissaList
293 noise = dataOut.noise
293 noise = dataOut.noise
294 SNR = dataOut.data_SNR
294 SNR = dataOut.data_SNR
295 nChannels = dataOut.nChannels
295 nChannels = dataOut.nChannels
296 # pairsList = dataOut.groupList
296 # pairsList = dataOut.groupList
297 # pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairsList, nChannels)
297 # pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairsList, nChannels)
298
298
299 for l in range(len(pairs_acf)):
299 for l in range(len(pairs_acf)):
300 data_acf[l,:,:] = data_acf[l,:,:]/normFactor_acf[l,:]
300 data_acf[l,:,:] = data_acf[l,:,:]/normFactor_acf[l,:]
301
301
302 for l in range(len(pairs_ccf)):
302 for l in range(len(pairs_ccf)):
303 data_ccf[l,:,:] = data_ccf[l,:,:]/normFactor_ccf[l,:]
303 data_ccf[l,:,:] = data_ccf[l,:,:]/normFactor_ccf[l,:]
304
304
305 dataOut.data_param = numpy.zeros((len(pairs_ccf)*2 + 1, nHeights))
305 dataOut.data_param = numpy.zeros((len(pairs_ccf)*2 + 1, nHeights))
306 dataOut.data_param[:-1,:] = self.__calculateTaus(data_acf, data_ccf, absc)
306 dataOut.data_param[:-1,:] = self.__calculateTaus(data_acf, data_ccf, absc)
307 dataOut.data_param[-1,:] = self.__calculateLag1Phase(data_acf, absc)
307 dataOut.data_param[-1,:] = self.__calculateLag1Phase(data_acf, absc)
308 return
308 return
309
309
310 # def __getPairsAutoCorr(self, pairsList, nChannels):
310 # def __getPairsAutoCorr(self, pairsList, nChannels):
311 #
311 #
312 # pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
312 # pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
313 #
313 #
314 # for l in range(len(pairsList)):
314 # for l in range(len(pairsList)):
315 # firstChannel = pairsList[l][0]
315 # firstChannel = pairsList[l][0]
316 # secondChannel = pairsList[l][1]
316 # secondChannel = pairsList[l][1]
317 #
317 #
318 # #Obteniendo pares de Autocorrelacion
318 # #Obteniendo pares de Autocorrelacion
319 # if firstChannel == secondChannel:
319 # if firstChannel == secondChannel:
320 # pairsAutoCorr[firstChannel] = int(l)
320 # pairsAutoCorr[firstChannel] = int(l)
321 #
321 #
322 # pairsAutoCorr = pairsAutoCorr.astype(int)
322 # pairsAutoCorr = pairsAutoCorr.astype(int)
323 #
323 #
324 # pairsCrossCorr = range(len(pairsList))
324 # pairsCrossCorr = range(len(pairsList))
325 # pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
325 # pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
326 #
326 #
327 # return pairsAutoCorr, pairsCrossCorr
327 # return pairsAutoCorr, pairsCrossCorr
328
328
329 def __calculateTaus(self, data_acf, data_ccf, lagRange):
329 def __calculateTaus(self, data_acf, data_ccf, lagRange):
330
330
331 lag0 = data_acf.shape[1]/2
331 lag0 = data_acf.shape[1]/2
332 #Funcion de Autocorrelacion
332 #Funcion de Autocorrelacion
333 mean_acf = stats.nanmean(data_acf, axis = 0)
333 mean_acf = stats.nanmean(data_acf, axis = 0)
334
334
335 #Obtencion Indice de TauCross
335 #Obtencion Indice de TauCross
336 ind_ccf = data_ccf.argmax(axis = 1)
336 ind_ccf = data_ccf.argmax(axis = 1)
337 #Obtencion Indice de TauAuto
337 #Obtencion Indice de TauAuto
338 ind_acf = numpy.zeros(ind_ccf.shape,dtype = 'int')
338 ind_acf = numpy.zeros(ind_ccf.shape,dtype = 'int')
339 ccf_lag0 = data_ccf[:,lag0,:]
339 ccf_lag0 = data_ccf[:,lag0,:]
340
340
341 for i in range(ccf_lag0.shape[0]):
341 for i in range(ccf_lag0.shape[0]):
342 ind_acf[i,:] = numpy.abs(mean_acf - ccf_lag0[i,:]).argmin(axis = 0)
342 ind_acf[i,:] = numpy.abs(mean_acf - ccf_lag0[i,:]).argmin(axis = 0)
343
343
344 #Obtencion de TauCross y TauAuto
344 #Obtencion de TauCross y TauAuto
345 tau_ccf = lagRange[ind_ccf]
345 tau_ccf = lagRange[ind_ccf]
346 tau_acf = lagRange[ind_acf]
346 tau_acf = lagRange[ind_acf]
347
347
348 Nan1, Nan2 = numpy.where(tau_ccf == lagRange[0])
348 Nan1, Nan2 = numpy.where(tau_ccf == lagRange[0])
349
349
350 tau_ccf[Nan1,Nan2] = numpy.nan
350 tau_ccf[Nan1,Nan2] = numpy.nan
351 tau_acf[Nan1,Nan2] = numpy.nan
351 tau_acf[Nan1,Nan2] = numpy.nan
352 tau = numpy.vstack((tau_ccf,tau_acf))
352 tau = numpy.vstack((tau_ccf,tau_acf))
353
353
354 return tau
354 return tau
355
355
356 def __calculateLag1Phase(self, data, lagTRange):
356 def __calculateLag1Phase(self, data, lagTRange):
357 data1 = stats.nanmean(data, axis = 0)
357 data1 = stats.nanmean(data, axis = 0)
358 lag1 = numpy.where(lagTRange == 0)[0][0] + 1
358 lag1 = numpy.where(lagTRange == 0)[0][0] + 1
359
359
360 phase = numpy.angle(data1[lag1,:])
360 phase = numpy.angle(data1[lag1,:])
361
361
362 return phase
362 return phase
363
363
364 class SpectralFitting(Operation):
364 class SpectralFitting(Operation):
365 '''
365 '''
366 Function GetMoments()
366 Function GetMoments()
367
367
368 Input:
368 Input:
369 Output:
369 Output:
370 Variables modified:
370 Variables modified:
371 '''
371 '''
372
372
373 def run(self, dataOut, getSNR = True, path=None, file=None, groupList=None):
373 def run(self, dataOut, getSNR = True, path=None, file=None, groupList=None):
374
374
375
375
376 if path != None:
376 if path != None:
377 sys.path.append(path)
377 sys.path.append(path)
378 self.dataOut.library = importlib.import_module(file)
378 self.dataOut.library = importlib.import_module(file)
379
379
380 #To be inserted as a parameter
380 #To be inserted as a parameter
381 groupArray = numpy.array(groupList)
381 groupArray = numpy.array(groupList)
382 # groupArray = numpy.array([[0,1],[2,3]])
382 # groupArray = numpy.array([[0,1],[2,3]])
383 self.dataOut.groupList = groupArray
383 self.dataOut.groupList = groupArray
384
384
385 nGroups = groupArray.shape[0]
385 nGroups = groupArray.shape[0]
386 nChannels = self.dataIn.nChannels
386 nChannels = self.dataIn.nChannels
387 nHeights=self.dataIn.heightList.size
387 nHeights=self.dataIn.heightList.size
388
388
389 #Parameters Array
389 #Parameters Array
390 self.dataOut.data_param = None
390 self.dataOut.data_param = None
391
391
392 #Set constants
392 #Set constants
393 constants = self.dataOut.library.setConstants(self.dataIn)
393 constants = self.dataOut.library.setConstants(self.dataIn)
394 self.dataOut.constants = constants
394 self.dataOut.constants = constants
395 M = self.dataIn.normFactor
395 M = self.dataIn.normFactor
396 N = self.dataIn.nFFTPoints
396 N = self.dataIn.nFFTPoints
397 ippSeconds = self.dataIn.ippSeconds
397 ippSeconds = self.dataIn.ippSeconds
398 K = self.dataIn.nIncohInt
398 K = self.dataIn.nIncohInt
399 pairsArray = numpy.array(self.dataIn.pairsList)
399 pairsArray = numpy.array(self.dataIn.pairsList)
400
400
401 #List of possible combinations
401 #List of possible combinations
402 listComb = itertools.combinations(numpy.arange(groupArray.shape[1]),2)
402 listComb = itertools.combinations(numpy.arange(groupArray.shape[1]),2)
403 indCross = numpy.zeros(len(list(listComb)), dtype = 'int')
403 indCross = numpy.zeros(len(list(listComb)), dtype = 'int')
404
404
405 if getSNR:
405 if getSNR:
406 listChannels = groupArray.reshape((groupArray.size))
406 listChannels = groupArray.reshape((groupArray.size))
407 listChannels.sort()
407 listChannels.sort()
408 noise = self.dataIn.getNoise()
408 noise = self.dataIn.getNoise()
409 self.dataOut.data_SNR = self.__getSNR(self.dataIn.data_spc[listChannels,:,:], noise[listChannels])
409 self.dataOut.data_SNR = self.__getSNR(self.dataIn.data_spc[listChannels,:,:], noise[listChannels])
410
410
411 for i in range(nGroups):
411 for i in range(nGroups):
412 coord = groupArray[i,:]
412 coord = groupArray[i,:]
413
413
414 #Input data array
414 #Input data array
415 data = self.dataIn.data_spc[coord,:,:]/(M*N)
415 data = self.dataIn.data_spc[coord,:,:]/(M*N)
416 data = data.reshape((data.shape[0]*data.shape[1],data.shape[2]))
416 data = data.reshape((data.shape[0]*data.shape[1],data.shape[2]))
417
417
418 #Cross Spectra data array for Covariance Matrixes
418 #Cross Spectra data array for Covariance Matrixes
419 ind = 0
419 ind = 0
420 for pairs in listComb:
420 for pairs in listComb:
421 pairsSel = numpy.array([coord[x],coord[y]])
421 pairsSel = numpy.array([coord[x],coord[y]])
422 indCross[ind] = int(numpy.where(numpy.all(pairsArray == pairsSel, axis = 1))[0][0])
422 indCross[ind] = int(numpy.where(numpy.all(pairsArray == pairsSel, axis = 1))[0][0])
423 ind += 1
423 ind += 1
424 dataCross = self.dataIn.data_cspc[indCross,:,:]/(M*N)
424 dataCross = self.dataIn.data_cspc[indCross,:,:]/(M*N)
425 dataCross = dataCross**2/K
425 dataCross = dataCross**2/K
426
426
427 for h in range(nHeights):
427 for h in range(nHeights):
428 # print self.dataOut.heightList[h]
428 # print self.dataOut.heightList[h]
429
429
430 #Input
430 #Input
431 d = data[:,h]
431 d = data[:,h]
432
432
433 #Covariance Matrix
433 #Covariance Matrix
434 D = numpy.diag(d**2/K)
434 D = numpy.diag(d**2/K)
435 ind = 0
435 ind = 0
436 for pairs in listComb:
436 for pairs in listComb:
437 #Coordinates in Covariance Matrix
437 #Coordinates in Covariance Matrix
438 x = pairs[0]
438 x = pairs[0]
439 y = pairs[1]
439 y = pairs[1]
440 #Channel Index
440 #Channel Index
441 S12 = dataCross[ind,:,h]
441 S12 = dataCross[ind,:,h]
442 D12 = numpy.diag(S12)
442 D12 = numpy.diag(S12)
443 #Completing Covariance Matrix with Cross Spectras
443 #Completing Covariance Matrix with Cross Spectras
444 D[x*N:(x+1)*N,y*N:(y+1)*N] = D12
444 D[x*N:(x+1)*N,y*N:(y+1)*N] = D12
445 D[y*N:(y+1)*N,x*N:(x+1)*N] = D12
445 D[y*N:(y+1)*N,x*N:(x+1)*N] = D12
446 ind += 1
446 ind += 1
447 Dinv=numpy.linalg.inv(D)
447 Dinv=numpy.linalg.inv(D)
448 L=numpy.linalg.cholesky(Dinv)
448 L=numpy.linalg.cholesky(Dinv)
449 LT=L.T
449 LT=L.T
450
450
451 dp = numpy.dot(LT,d)
451 dp = numpy.dot(LT,d)
452
452
453 #Initial values
453 #Initial values
454 data_spc = self.dataIn.data_spc[coord,:,h]
454 data_spc = self.dataIn.data_spc[coord,:,h]
455
455
456 if (h>0)and(error1[3]<5):
456 if (h>0)and(error1[3]<5):
457 p0 = self.dataOut.data_param[i,:,h-1]
457 p0 = self.dataOut.data_param[i,:,h-1]
458 else:
458 else:
459 p0 = numpy.array(self.dataOut.library.initialValuesFunction(data_spc, constants, i))
459 p0 = numpy.array(self.dataOut.library.initialValuesFunction(data_spc, constants, i))
460
460
461 try:
461 try:
462 #Least Squares
462 #Least Squares
463 minp,covp,infodict,mesg,ier = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants),full_output=True)
463 minp,covp,infodict,mesg,ier = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants),full_output=True)
464 # minp,covp = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants))
464 # minp,covp = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants))
465 #Chi square error
465 #Chi square error
466 error0 = numpy.sum(infodict['fvec']**2)/(2*N)
466 error0 = numpy.sum(infodict['fvec']**2)/(2*N)
467 #Error with Jacobian
467 #Error with Jacobian
468 error1 = self.dataOut.library.errorFunction(minp,constants,LT)
468 error1 = self.dataOut.library.errorFunction(minp,constants,LT)
469 except:
469 except:
470 minp = p0*numpy.nan
470 minp = p0*numpy.nan
471 error0 = numpy.nan
471 error0 = numpy.nan
472 error1 = p0*numpy.nan
472 error1 = p0*numpy.nan
473
473
474 #Save
474 #Save
475 if self.dataOut.data_param is None:
475 if self.dataOut.data_param is None:
476 self.dataOut.data_param = numpy.zeros((nGroups, p0.size, nHeights))*numpy.nan
476 self.dataOut.data_param = numpy.zeros((nGroups, p0.size, nHeights))*numpy.nan
477 self.dataOut.data_error = numpy.zeros((nGroups, p0.size + 1, nHeights))*numpy.nan
477 self.dataOut.data_error = numpy.zeros((nGroups, p0.size + 1, nHeights))*numpy.nan
478
478
479 self.dataOut.data_error[i,:,h] = numpy.hstack((error0,error1))
479 self.dataOut.data_error[i,:,h] = numpy.hstack((error0,error1))
480 self.dataOut.data_param[i,:,h] = minp
480 self.dataOut.data_param[i,:,h] = minp
481 return
481 return
482
482
483 def __residFunction(self, p, dp, LT, constants):
483 def __residFunction(self, p, dp, LT, constants):
484
484
485 fm = self.dataOut.library.modelFunction(p, constants)
485 fm = self.dataOut.library.modelFunction(p, constants)
486 fmp=numpy.dot(LT,fm)
486 fmp=numpy.dot(LT,fm)
487
487
488 return dp-fmp
488 return dp-fmp
489
489
490 def __getSNR(self, z, noise):
490 def __getSNR(self, z, noise):
491
491
492 avg = numpy.average(z, axis=1)
492 avg = numpy.average(z, axis=1)
493 SNR = (avg.T-noise)/noise
493 SNR = (avg.T-noise)/noise
494 SNR = SNR.T
494 SNR = SNR.T
495 return SNR
495 return SNR
496
496
497 def __chisq(p,chindex,hindex):
497 def __chisq(p,chindex,hindex):
498 #similar to Resid but calculates CHI**2
498 #similar to Resid but calculates CHI**2
499 [LT,d,fm]=setupLTdfm(p,chindex,hindex)
499 [LT,d,fm]=setupLTdfm(p,chindex,hindex)
500 dp=numpy.dot(LT,d)
500 dp=numpy.dot(LT,d)
501 fmp=numpy.dot(LT,fm)
501 fmp=numpy.dot(LT,fm)
502 chisq=numpy.dot((dp-fmp).T,(dp-fmp))
502 chisq=numpy.dot((dp-fmp).T,(dp-fmp))
503 return chisq
503 return chisq
504
504
505 class WindProfiler(Operation):
505 class WindProfiler(Operation):
506
506
507 __isConfig = False
507 __isConfig = False
508
508
509 __initime = None
509 __initime = None
510 __lastdatatime = None
510 __lastdatatime = None
511 __integrationtime = None
511 __integrationtime = None
512
512
513 __buffer = None
513 __buffer = None
514
514
515 __dataReady = False
515 __dataReady = False
516
516
517 __firstdata = None
517 __firstdata = None
518
518
519 n = None
519 n = None
520
520
521 def __init__(self):
522 Operation.__init__(self)
523
524 def __calculateCosDir(self, elev, azim):
521 def __calculateCosDir(self, elev, azim):
525 zen = (90 - elev)*numpy.pi/180
522 zen = (90 - elev)*numpy.pi/180
526 azim = azim*numpy.pi/180
523 azim = azim*numpy.pi/180
527 cosDirX = numpy.sqrt((1-numpy.cos(zen)**2)/((1+numpy.tan(azim)**2)))
524 cosDirX = numpy.sqrt((1-numpy.cos(zen)**2)/((1+numpy.tan(azim)**2)))
528 cosDirY = numpy.sqrt(1-numpy.cos(zen)**2-cosDirX**2)
525 cosDirY = numpy.sqrt(1-numpy.cos(zen)**2-cosDirX**2)
529
526
530 signX = numpy.sign(numpy.cos(azim))
527 signX = numpy.sign(numpy.cos(azim))
531 signY = numpy.sign(numpy.sin(azim))
528 signY = numpy.sign(numpy.sin(azim))
532
529
533 cosDirX = numpy.copysign(cosDirX, signX)
530 cosDirX = numpy.copysign(cosDirX, signX)
534 cosDirY = numpy.copysign(cosDirY, signY)
531 cosDirY = numpy.copysign(cosDirY, signY)
535 return cosDirX, cosDirY
532 return cosDirX, cosDirY
536
533
537 def __calculateAngles(self, theta_x, theta_y, azimuth):
534 def __calculateAngles(self, theta_x, theta_y, azimuth):
538
535
539 dir_cosw = numpy.sqrt(1-theta_x**2-theta_y**2)
536 dir_cosw = numpy.sqrt(1-theta_x**2-theta_y**2)
540 zenith_arr = numpy.arccos(dir_cosw)
537 zenith_arr = numpy.arccos(dir_cosw)
541 azimuth_arr = numpy.arctan2(theta_x,theta_y) + azimuth*math.pi/180
538 azimuth_arr = numpy.arctan2(theta_x,theta_y) + azimuth*math.pi/180
542
539
543 dir_cosu = numpy.sin(azimuth_arr)*numpy.sin(zenith_arr)
540 dir_cosu = numpy.sin(azimuth_arr)*numpy.sin(zenith_arr)
544 dir_cosv = numpy.cos(azimuth_arr)*numpy.sin(zenith_arr)
541 dir_cosv = numpy.cos(azimuth_arr)*numpy.sin(zenith_arr)
545
542
546 return azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw
543 return azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw
547
544
548 def __calculateMatA(self, dir_cosu, dir_cosv, dir_cosw, horOnly):
545 def __calculateMatA(self, dir_cosu, dir_cosv, dir_cosw, horOnly):
549
546
550 #
547 #
551 if horOnly:
548 if horOnly:
552 A = numpy.c_[dir_cosu,dir_cosv]
549 A = numpy.c_[dir_cosu,dir_cosv]
553 else:
550 else:
554 A = numpy.c_[dir_cosu,dir_cosv,dir_cosw]
551 A = numpy.c_[dir_cosu,dir_cosv,dir_cosw]
555 A = numpy.asmatrix(A)
552 A = numpy.asmatrix(A)
556 A1 = numpy.linalg.inv(A.transpose()*A)*A.transpose()
553 A1 = numpy.linalg.inv(A.transpose()*A)*A.transpose()
557
554
558 return A1
555 return A1
559
556
560 def __correctValues(self, heiRang, phi, velRadial, SNR):
557 def __correctValues(self, heiRang, phi, velRadial, SNR):
561 listPhi = phi.tolist()
558 listPhi = phi.tolist()
562 maxid = listPhi.index(max(listPhi))
559 maxid = listPhi.index(max(listPhi))
563 minid = listPhi.index(min(listPhi))
560 minid = listPhi.index(min(listPhi))
564
561
565 rango = range(len(phi))
562 rango = range(len(phi))
566 # rango = numpy.delete(rango,maxid)
563 # rango = numpy.delete(rango,maxid)
567
564
568 heiRang1 = heiRang*math.cos(phi[maxid])
565 heiRang1 = heiRang*math.cos(phi[maxid])
569 heiRangAux = heiRang*math.cos(phi[minid])
566 heiRangAux = heiRang*math.cos(phi[minid])
570 indOut = (heiRang1 < heiRangAux[0]).nonzero()
567 indOut = (heiRang1 < heiRangAux[0]).nonzero()
571 heiRang1 = numpy.delete(heiRang1,indOut)
568 heiRang1 = numpy.delete(heiRang1,indOut)
572
569
573 velRadial1 = numpy.zeros([len(phi),len(heiRang1)])
570 velRadial1 = numpy.zeros([len(phi),len(heiRang1)])
574 SNR1 = numpy.zeros([len(phi),len(heiRang1)])
571 SNR1 = numpy.zeros([len(phi),len(heiRang1)])
575
572
576 for i in rango:
573 for i in rango:
577 x = heiRang*math.cos(phi[i])
574 x = heiRang*math.cos(phi[i])
578 y1 = velRadial[i,:]
575 y1 = velRadial[i,:]
579 f1 = interpolate.interp1d(x,y1,kind = 'cubic')
576 f1 = interpolate.interp1d(x,y1,kind = 'cubic')
580
577
581 x1 = heiRang1
578 x1 = heiRang1
582 y11 = f1(x1)
579 y11 = f1(x1)
583
580
584 y2 = SNR[i,:]
581 y2 = SNR[i,:]
585 f2 = interpolate.interp1d(x,y2,kind = 'cubic')
582 f2 = interpolate.interp1d(x,y2,kind = 'cubic')
586 y21 = f2(x1)
583 y21 = f2(x1)
587
584
588 velRadial1[i,:] = y11
585 velRadial1[i,:] = y11
589 SNR1[i,:] = y21
586 SNR1[i,:] = y21
590
587
591 return heiRang1, velRadial1, SNR1
588 return heiRang1, velRadial1, SNR1
592
589
593 def __calculateVelUVW(self, A, velRadial):
590 def __calculateVelUVW(self, A, velRadial):
594
591
595 #Operacion Matricial
592 #Operacion Matricial
596 # velUVW = numpy.zeros((velRadial.shape[1],3))
593 # velUVW = numpy.zeros((velRadial.shape[1],3))
597 # for ind in range(velRadial.shape[1]):
594 # for ind in range(velRadial.shape[1]):
598 # velUVW[ind,:] = numpy.dot(A,velRadial[:,ind])
595 # velUVW[ind,:] = numpy.dot(A,velRadial[:,ind])
599 # velUVW = velUVW.transpose()
596 # velUVW = velUVW.transpose()
600 velUVW = numpy.zeros((A.shape[0],velRadial.shape[1]))
597 velUVW = numpy.zeros((A.shape[0],velRadial.shape[1]))
601 velUVW[:,:] = numpy.dot(A,velRadial)
598 velUVW[:,:] = numpy.dot(A,velRadial)
602
599
603
600
604 return velUVW
601 return velUVW
605
602
606 # def techniqueDBS(self, velRadial0, dirCosx, disrCosy, azimuth, correct, horizontalOnly, heiRang, SNR0):
603 # def techniqueDBS(self, velRadial0, dirCosx, disrCosy, azimuth, correct, horizontalOnly, heiRang, SNR0):
607
604
608 def techniqueDBS(self, kwargs):
605 def techniqueDBS(self, kwargs):
609 """
606 """
610 Function that implements Doppler Beam Swinging (DBS) technique.
607 Function that implements Doppler Beam Swinging (DBS) technique.
611
608
612 Input: Radial velocities, Direction cosines (x and y) of the Beam, Antenna azimuth,
609 Input: Radial velocities, Direction cosines (x and y) of the Beam, Antenna azimuth,
613 Direction correction (if necessary), Ranges and SNR
610 Direction correction (if necessary), Ranges and SNR
614
611
615 Output: Winds estimation (Zonal, Meridional and Vertical)
612 Output: Winds estimation (Zonal, Meridional and Vertical)
616
613
617 Parameters affected: Winds, height range, SNR
614 Parameters affected: Winds, height range, SNR
618 """
615 """
619 velRadial0 = kwargs['velRadial']
616 velRadial0 = kwargs['velRadial']
620 heiRang = kwargs['heightList']
617 heiRang = kwargs['heightList']
621 SNR0 = kwargs['SNR']
618 SNR0 = kwargs['SNR']
622
619
623 if kwargs.has_key('dirCosx') and kwargs.has_key('dirCosy'):
620 if kwargs.has_key('dirCosx') and kwargs.has_key('dirCosy'):
624 theta_x = numpy.array(kwargs['dirCosx'])
621 theta_x = numpy.array(kwargs['dirCosx'])
625 theta_y = numpy.array(kwargs['dirCosy'])
622 theta_y = numpy.array(kwargs['dirCosy'])
626 else:
623 else:
627 elev = numpy.array(kwargs['elevation'])
624 elev = numpy.array(kwargs['elevation'])
628 azim = numpy.array(kwargs['azimuth'])
625 azim = numpy.array(kwargs['azimuth'])
629 theta_x, theta_y = self.__calculateCosDir(elev, azim)
626 theta_x, theta_y = self.__calculateCosDir(elev, azim)
630 azimuth = kwargs['correctAzimuth']
627 azimuth = kwargs['correctAzimuth']
631 if kwargs.has_key('horizontalOnly'):
628 if kwargs.has_key('horizontalOnly'):
632 horizontalOnly = kwargs['horizontalOnly']
629 horizontalOnly = kwargs['horizontalOnly']
633 else: horizontalOnly = False
630 else: horizontalOnly = False
634 if kwargs.has_key('correctFactor'):
631 if kwargs.has_key('correctFactor'):
635 correctFactor = kwargs['correctFactor']
632 correctFactor = kwargs['correctFactor']
636 else: correctFactor = 1
633 else: correctFactor = 1
637 if kwargs.has_key('channelList'):
634 if kwargs.has_key('channelList'):
638 channelList = kwargs['channelList']
635 channelList = kwargs['channelList']
639 if len(channelList) == 2:
636 if len(channelList) == 2:
640 horizontalOnly = True
637 horizontalOnly = True
641 arrayChannel = numpy.array(channelList)
638 arrayChannel = numpy.array(channelList)
642 param = param[arrayChannel,:,:]
639 param = param[arrayChannel,:,:]
643 theta_x = theta_x[arrayChannel]
640 theta_x = theta_x[arrayChannel]
644 theta_y = theta_y[arrayChannel]
641 theta_y = theta_y[arrayChannel]
645
642
646 azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw = self.__calculateAngles(theta_x, theta_y, azimuth)
643 azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw = self.__calculateAngles(theta_x, theta_y, azimuth)
647 heiRang1, velRadial1, SNR1 = self.__correctValues(heiRang, zenith_arr, correctFactor*velRadial0, SNR0)
644 heiRang1, velRadial1, SNR1 = self.__correctValues(heiRang, zenith_arr, correctFactor*velRadial0, SNR0)
648 A = self.__calculateMatA(dir_cosu, dir_cosv, dir_cosw, horizontalOnly)
645 A = self.__calculateMatA(dir_cosu, dir_cosv, dir_cosw, horizontalOnly)
649
646
650 #Calculo de Componentes de la velocidad con DBS
647 #Calculo de Componentes de la velocidad con DBS
651 winds = self.__calculateVelUVW(A,velRadial1)
648 winds = self.__calculateVelUVW(A,velRadial1)
652
649
653 return winds, heiRang1, SNR1
650 return winds, heiRang1, SNR1
654
651
655 def __calculateDistance(self, posx, posy, pairs_ccf, azimuth = None):
652 def __calculateDistance(self, posx, posy, pairs_ccf, azimuth = None):
656
653
657 nPairs = len(pairs_ccf)
654 nPairs = len(pairs_ccf)
658 posx = numpy.asarray(posx)
655 posx = numpy.asarray(posx)
659 posy = numpy.asarray(posy)
656 posy = numpy.asarray(posy)
660
657
661 #Rotacion Inversa para alinear con el azimuth
658 #Rotacion Inversa para alinear con el azimuth
662 if azimuth!= None:
659 if azimuth!= None:
663 azimuth = azimuth*math.pi/180
660 azimuth = azimuth*math.pi/180
664 posx1 = posx*math.cos(azimuth) + posy*math.sin(azimuth)
661 posx1 = posx*math.cos(azimuth) + posy*math.sin(azimuth)
665 posy1 = -posx*math.sin(azimuth) + posy*math.cos(azimuth)
662 posy1 = -posx*math.sin(azimuth) + posy*math.cos(azimuth)
666 else:
663 else:
667 posx1 = posx
664 posx1 = posx
668 posy1 = posy
665 posy1 = posy
669
666
670 #Calculo de Distancias
667 #Calculo de Distancias
671 distx = numpy.zeros(nPairs)
668 distx = numpy.zeros(nPairs)
672 disty = numpy.zeros(nPairs)
669 disty = numpy.zeros(nPairs)
673 dist = numpy.zeros(nPairs)
670 dist = numpy.zeros(nPairs)
674 ang = numpy.zeros(nPairs)
671 ang = numpy.zeros(nPairs)
675
672
676 for i in range(nPairs):
673 for i in range(nPairs):
677 distx[i] = posx1[pairs_ccf[i][1]] - posx1[pairs_ccf[i][0]]
674 distx[i] = posx1[pairs_ccf[i][1]] - posx1[pairs_ccf[i][0]]
678 disty[i] = posy1[pairs_ccf[i][1]] - posy1[pairs_ccf[i][0]]
675 disty[i] = posy1[pairs_ccf[i][1]] - posy1[pairs_ccf[i][0]]
679 dist[i] = numpy.sqrt(distx[i]**2 + disty[i]**2)
676 dist[i] = numpy.sqrt(distx[i]**2 + disty[i]**2)
680 ang[i] = numpy.arctan2(disty[i],distx[i])
677 ang[i] = numpy.arctan2(disty[i],distx[i])
681
678
682 return distx, disty, dist, ang
679 return distx, disty, dist, ang
683 #Calculo de Matrices
680 #Calculo de Matrices
684 # nPairs = len(pairs)
681 # nPairs = len(pairs)
685 # ang1 = numpy.zeros((nPairs, 2, 1))
682 # ang1 = numpy.zeros((nPairs, 2, 1))
686 # dist1 = numpy.zeros((nPairs, 2, 1))
683 # dist1 = numpy.zeros((nPairs, 2, 1))
687 #
684 #
688 # for j in range(nPairs):
685 # for j in range(nPairs):
689 # dist1[j,0,0] = dist[pairs[j][0]]
686 # dist1[j,0,0] = dist[pairs[j][0]]
690 # dist1[j,1,0] = dist[pairs[j][1]]
687 # dist1[j,1,0] = dist[pairs[j][1]]
691 # ang1[j,0,0] = ang[pairs[j][0]]
688 # ang1[j,0,0] = ang[pairs[j][0]]
692 # ang1[j,1,0] = ang[pairs[j][1]]
689 # ang1[j,1,0] = ang[pairs[j][1]]
693 #
690 #
694 # return distx,disty, dist1,ang1
691 # return distx,disty, dist1,ang1
695
692
696
693
697 def __calculateVelVer(self, phase, lagTRange, _lambda):
694 def __calculateVelVer(self, phase, lagTRange, _lambda):
698
695
699 Ts = lagTRange[1] - lagTRange[0]
696 Ts = lagTRange[1] - lagTRange[0]
700 velW = -_lambda*phase/(4*math.pi*Ts)
697 velW = -_lambda*phase/(4*math.pi*Ts)
701
698
702 return velW
699 return velW
703
700
704 def __calculateVelHorDir(self, dist, tau1, tau2, ang):
701 def __calculateVelHorDir(self, dist, tau1, tau2, ang):
705 nPairs = tau1.shape[0]
702 nPairs = tau1.shape[0]
706 nHeights = tau1.shape[1]
703 nHeights = tau1.shape[1]
707 vel = numpy.zeros((nPairs,3,nHeights))
704 vel = numpy.zeros((nPairs,3,nHeights))
708 dist1 = numpy.reshape(dist, (dist.size,1))
705 dist1 = numpy.reshape(dist, (dist.size,1))
709
706
710 angCos = numpy.cos(ang)
707 angCos = numpy.cos(ang)
711 angSin = numpy.sin(ang)
708 angSin = numpy.sin(ang)
712
709
713 vel0 = dist1*tau1/(2*tau2**2)
710 vel0 = dist1*tau1/(2*tau2**2)
714 vel[:,0,:] = (vel0*angCos).sum(axis = 1)
711 vel[:,0,:] = (vel0*angCos).sum(axis = 1)
715 vel[:,1,:] = (vel0*angSin).sum(axis = 1)
712 vel[:,1,:] = (vel0*angSin).sum(axis = 1)
716
713
717 ind = numpy.where(numpy.isinf(vel))
714 ind = numpy.where(numpy.isinf(vel))
718 vel[ind] = numpy.nan
715 vel[ind] = numpy.nan
719
716
720 return vel
717 return vel
721
718
722 # def __getPairsAutoCorr(self, pairsList, nChannels):
719 # def __getPairsAutoCorr(self, pairsList, nChannels):
723 #
720 #
724 # pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
721 # pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
725 #
722 #
726 # for l in range(len(pairsList)):
723 # for l in range(len(pairsList)):
727 # firstChannel = pairsList[l][0]
724 # firstChannel = pairsList[l][0]
728 # secondChannel = pairsList[l][1]
725 # secondChannel = pairsList[l][1]
729 #
726 #
730 # #Obteniendo pares de Autocorrelacion
727 # #Obteniendo pares de Autocorrelacion
731 # if firstChannel == secondChannel:
728 # if firstChannel == secondChannel:
732 # pairsAutoCorr[firstChannel] = int(l)
729 # pairsAutoCorr[firstChannel] = int(l)
733 #
730 #
734 # pairsAutoCorr = pairsAutoCorr.astype(int)
731 # pairsAutoCorr = pairsAutoCorr.astype(int)
735 #
732 #
736 # pairsCrossCorr = range(len(pairsList))
733 # pairsCrossCorr = range(len(pairsList))
737 # pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
734 # pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
738 #
735 #
739 # return pairsAutoCorr, pairsCrossCorr
736 # return pairsAutoCorr, pairsCrossCorr
740
737
741 # def techniqueSA(self, pairsSelected, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, lagTRange, correctFactor):
738 # def techniqueSA(self, pairsSelected, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, lagTRange, correctFactor):
742 def techniqueSA(self, kwargs):
739 def techniqueSA(self, kwargs):
743
740
744 """
741 """
745 Function that implements Spaced Antenna (SA) technique.
742 Function that implements Spaced Antenna (SA) technique.
746
743
747 Input: Radial velocities, Direction cosines (x and y) of the Beam, Antenna azimuth,
744 Input: Radial velocities, Direction cosines (x and y) of the Beam, Antenna azimuth,
748 Direction correction (if necessary), Ranges and SNR
745 Direction correction (if necessary), Ranges and SNR
749
746
750 Output: Winds estimation (Zonal, Meridional and Vertical)
747 Output: Winds estimation (Zonal, Meridional and Vertical)
751
748
752 Parameters affected: Winds
749 Parameters affected: Winds
753 """
750 """
754 position_x = kwargs['positionX']
751 position_x = kwargs['positionX']
755 position_y = kwargs['positionY']
752 position_y = kwargs['positionY']
756 azimuth = kwargs['azimuth']
753 azimuth = kwargs['azimuth']
757
754
758 if kwargs.has_key('correctFactor'):
755 if kwargs.has_key('correctFactor'):
759 correctFactor = kwargs['correctFactor']
756 correctFactor = kwargs['correctFactor']
760 else:
757 else:
761 correctFactor = 1
758 correctFactor = 1
762
759
763 groupList = kwargs['groupList']
760 groupList = kwargs['groupList']
764 pairs_ccf = groupList[1]
761 pairs_ccf = groupList[1]
765 tau = kwargs['tau']
762 tau = kwargs['tau']
766 _lambda = kwargs['_lambda']
763 _lambda = kwargs['_lambda']
767
764
768 #Cross Correlation pairs obtained
765 #Cross Correlation pairs obtained
769 # pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairssList, nChannels)
766 # pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairssList, nChannels)
770 # pairsArray = numpy.array(pairsList)[pairsCrossCorr]
767 # pairsArray = numpy.array(pairsList)[pairsCrossCorr]
771 # pairsSelArray = numpy.array(pairsSelected)
768 # pairsSelArray = numpy.array(pairsSelected)
772 # pairs = []
769 # pairs = []
773 #
770 #
774 # #Wind estimation pairs obtained
771 # #Wind estimation pairs obtained
775 # for i in range(pairsSelArray.shape[0]/2):
772 # for i in range(pairsSelArray.shape[0]/2):
776 # ind1 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i], axis = 1))[0][0]
773 # ind1 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i], axis = 1))[0][0]
777 # ind2 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i + 1], axis = 1))[0][0]
774 # ind2 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i + 1], axis = 1))[0][0]
778 # pairs.append((ind1,ind2))
775 # pairs.append((ind1,ind2))
779
776
780 indtau = tau.shape[0]/2
777 indtau = tau.shape[0]/2
781 tau1 = tau[:indtau,:]
778 tau1 = tau[:indtau,:]
782 tau2 = tau[indtau:-1,:]
779 tau2 = tau[indtau:-1,:]
783 # tau1 = tau1[pairs,:]
780 # tau1 = tau1[pairs,:]
784 # tau2 = tau2[pairs,:]
781 # tau2 = tau2[pairs,:]
785 phase1 = tau[-1,:]
782 phase1 = tau[-1,:]
786
783
787 #---------------------------------------------------------------------
784 #---------------------------------------------------------------------
788 #Metodo Directo
785 #Metodo Directo
789 distx, disty, dist, ang = self.__calculateDistance(position_x, position_y, pairs_ccf,azimuth)
786 distx, disty, dist, ang = self.__calculateDistance(position_x, position_y, pairs_ccf,azimuth)
790 winds = self.__calculateVelHorDir(dist, tau1, tau2, ang)
787 winds = self.__calculateVelHorDir(dist, tau1, tau2, ang)
791 winds = stats.nanmean(winds, axis=0)
788 winds = stats.nanmean(winds, axis=0)
792 #---------------------------------------------------------------------
789 #---------------------------------------------------------------------
793 #Metodo General
790 #Metodo General
794 # distx, disty, dist = self.calculateDistance(position_x,position_y,pairsCrossCorr, pairsList, azimuth)
791 # distx, disty, dist = self.calculateDistance(position_x,position_y,pairsCrossCorr, pairsList, azimuth)
795 # #Calculo Coeficientes de Funcion de Correlacion
792 # #Calculo Coeficientes de Funcion de Correlacion
796 # F,G,A,B,H = self.calculateCoef(tau1,tau2,distx,disty,n)
793 # F,G,A,B,H = self.calculateCoef(tau1,tau2,distx,disty,n)
797 # #Calculo de Velocidades
794 # #Calculo de Velocidades
798 # winds = self.calculateVelUV(F,G,A,B,H)
795 # winds = self.calculateVelUV(F,G,A,B,H)
799
796
800 #---------------------------------------------------------------------
797 #---------------------------------------------------------------------
801 winds[2,:] = self.__calculateVelVer(phase1, lagTRange, _lambda)
798 winds[2,:] = self.__calculateVelVer(phase1, lagTRange, _lambda)
802 winds = correctFactor*winds
799 winds = correctFactor*winds
803 return winds
800 return winds
804
801
805 def __checkTime(self, currentTime, paramInterval, outputInterval):
802 def __checkTime(self, currentTime, paramInterval, outputInterval):
806
803
807 dataTime = currentTime + paramInterval
804 dataTime = currentTime + paramInterval
808 deltaTime = dataTime - self.__initime
805 deltaTime = dataTime - self.__initime
809
806
810 if deltaTime >= outputInterval or deltaTime < 0:
807 if deltaTime >= outputInterval or deltaTime < 0:
811 self.__dataReady = True
808 self.__dataReady = True
812 return
809 return
813
810
814 def techniqueMeteors(self, arrayMeteor, meteorThresh, heightMin, heightMax, binkm=2):
811 def techniqueMeteors(self, arrayMeteor, meteorThresh, heightMin, heightMax, binkm=2):
815 '''
812 '''
816 Function that implements winds estimation technique with detected meteors.
813 Function that implements winds estimation technique with detected meteors.
817
814
818 Input: Detected meteors, Minimum meteor quantity to wind estimation
815 Input: Detected meteors, Minimum meteor quantity to wind estimation
819
816
820 Output: Winds estimation (Zonal and Meridional)
817 Output: Winds estimation (Zonal and Meridional)
821
818
822 Parameters affected: Winds
819 Parameters affected: Winds
823 '''
820 '''
824 # print arrayMeteor.shape
821 # print arrayMeteor.shape
825 #Settings
822 #Settings
826 nInt = (heightMax - heightMin)/binkm
823 nInt = (heightMax - heightMin)/binkm
827 # print nInt
824 # print nInt
828 nInt = int(nInt)
825 nInt = int(nInt)
829 # print nInt
826 # print nInt
830 winds = numpy.zeros((2,nInt))*numpy.nan
827 winds = numpy.zeros((2,nInt))*numpy.nan
831
828
832 #Filter errors
829 #Filter errors
833 error = numpy.where(arrayMeteor[:,-1] == 0)[0]
830 error = numpy.where(arrayMeteor[:,-1] == 0)[0]
834 finalMeteor = arrayMeteor[error,:]
831 finalMeteor = arrayMeteor[error,:]
835
832
836 #Meteor Histogram
833 #Meteor Histogram
837 finalHeights = finalMeteor[:,2]
834 finalHeights = finalMeteor[:,2]
838 hist = numpy.histogram(finalHeights, bins = nInt, range = (heightMin,heightMax))
835 hist = numpy.histogram(finalHeights, bins = nInt, range = (heightMin,heightMax))
839 nMeteorsPerI = hist[0]
836 nMeteorsPerI = hist[0]
840 heightPerI = hist[1]
837 heightPerI = hist[1]
841
838
842 #Sort of meteors
839 #Sort of meteors
843 indSort = finalHeights.argsort()
840 indSort = finalHeights.argsort()
844 finalMeteor2 = finalMeteor[indSort,:]
841 finalMeteor2 = finalMeteor[indSort,:]
845
842
846 # Calculating winds
843 # Calculating winds
847 ind1 = 0
844 ind1 = 0
848 ind2 = 0
845 ind2 = 0
849
846
850 for i in range(nInt):
847 for i in range(nInt):
851 nMet = nMeteorsPerI[i]
848 nMet = nMeteorsPerI[i]
852 ind1 = ind2
849 ind1 = ind2
853 ind2 = ind1 + nMet
850 ind2 = ind1 + nMet
854
851
855 meteorAux = finalMeteor2[ind1:ind2,:]
852 meteorAux = finalMeteor2[ind1:ind2,:]
856
853
857 if meteorAux.shape[0] >= meteorThresh:
854 if meteorAux.shape[0] >= meteorThresh:
858 vel = meteorAux[:, 6]
855 vel = meteorAux[:, 6]
859 zen = meteorAux[:, 4]*numpy.pi/180
856 zen = meteorAux[:, 4]*numpy.pi/180
860 azim = meteorAux[:, 3]*numpy.pi/180
857 azim = meteorAux[:, 3]*numpy.pi/180
861
858
862 n = numpy.cos(zen)
859 n = numpy.cos(zen)
863 # m = (1 - n**2)/(1 - numpy.tan(azim)**2)
860 # m = (1 - n**2)/(1 - numpy.tan(azim)**2)
864 # l = m*numpy.tan(azim)
861 # l = m*numpy.tan(azim)
865 l = numpy.sin(zen)*numpy.sin(azim)
862 l = numpy.sin(zen)*numpy.sin(azim)
866 m = numpy.sin(zen)*numpy.cos(azim)
863 m = numpy.sin(zen)*numpy.cos(azim)
867
864
868 A = numpy.vstack((l, m)).transpose()
865 A = numpy.vstack((l, m)).transpose()
869 A1 = numpy.dot(numpy.linalg.inv( numpy.dot(A.transpose(),A) ),A.transpose())
866 A1 = numpy.dot(numpy.linalg.inv( numpy.dot(A.transpose(),A) ),A.transpose())
870 windsAux = numpy.dot(A1, vel)
867 windsAux = numpy.dot(A1, vel)
871
868
872 winds[0,i] = windsAux[0]
869 winds[0,i] = windsAux[0]
873 winds[1,i] = windsAux[1]
870 winds[1,i] = windsAux[1]
874
871
875 return winds, heightPerI[:-1]
872 return winds, heightPerI[:-1]
876
873
877 def techniqueNSM_SA(self, **kwargs):
874 def techniqueNSM_SA(self, **kwargs):
878 metArray = kwargs['metArray']
875 metArray = kwargs['metArray']
879 heightList = kwargs['heightList']
876 heightList = kwargs['heightList']
880 timeList = kwargs['timeList']
877 timeList = kwargs['timeList']
881
878
882 rx_location = kwargs['rx_location']
879 rx_location = kwargs['rx_location']
883 groupList = kwargs['groupList']
880 groupList = kwargs['groupList']
884 azimuth = kwargs['azimuth']
881 azimuth = kwargs['azimuth']
885 dfactor = kwargs['dfactor']
882 dfactor = kwargs['dfactor']
886 k = kwargs['k']
883 k = kwargs['k']
887
884
888 azimuth1, dist = self.__calculateAzimuth1(rx_location, groupList, azimuth)
885 azimuth1, dist = self.__calculateAzimuth1(rx_location, groupList, azimuth)
889 d = dist*dfactor
886 d = dist*dfactor
890 #Phase calculation
887 #Phase calculation
891 metArray1 = self.__getPhaseSlope(metArray, heightList, timeList)
888 metArray1 = self.__getPhaseSlope(metArray, heightList, timeList)
892
889
893 metArray1[:,-2] = metArray1[:,-2]*metArray1[:,2]*1000/(k*d[metArray1[:,1].astype(int)]) #angles into velocities
890 metArray1[:,-2] = metArray1[:,-2]*metArray1[:,2]*1000/(k*d[metArray1[:,1].astype(int)]) #angles into velocities
894
891
895 velEst = numpy.zeros((heightList.size,2))*numpy.nan
892 velEst = numpy.zeros((heightList.size,2))*numpy.nan
896 azimuth1 = azimuth1*numpy.pi/180
893 azimuth1 = azimuth1*numpy.pi/180
897
894
898 for i in range(heightList.size):
895 for i in range(heightList.size):
899 h = heightList[i]
896 h = heightList[i]
900 indH = numpy.where((metArray1[:,2] == h)&(numpy.abs(metArray1[:,-2]) < 100))[0]
897 indH = numpy.where((metArray1[:,2] == h)&(numpy.abs(metArray1[:,-2]) < 100))[0]
901 metHeight = metArray1[indH,:]
898 metHeight = metArray1[indH,:]
902 if metHeight.shape[0] >= 2:
899 if metHeight.shape[0] >= 2:
903 velAux = numpy.asmatrix(metHeight[:,-2]).T #Radial Velocities
900 velAux = numpy.asmatrix(metHeight[:,-2]).T #Radial Velocities
904 iazim = metHeight[:,1].astype(int)
901 iazim = metHeight[:,1].astype(int)
905 azimAux = numpy.asmatrix(azimuth1[iazim]).T #Azimuths
902 azimAux = numpy.asmatrix(azimuth1[iazim]).T #Azimuths
906 A = numpy.hstack((numpy.cos(azimAux),numpy.sin(azimAux)))
903 A = numpy.hstack((numpy.cos(azimAux),numpy.sin(azimAux)))
907 A = numpy.asmatrix(A)
904 A = numpy.asmatrix(A)
908 A1 = numpy.linalg.pinv(A.transpose()*A)*A.transpose()
905 A1 = numpy.linalg.pinv(A.transpose()*A)*A.transpose()
909 velHor = numpy.dot(A1,velAux)
906 velHor = numpy.dot(A1,velAux)
910
907
911 velEst[i,:] = numpy.squeeze(velHor)
908 velEst[i,:] = numpy.squeeze(velHor)
912 return velEst
909 return velEst
913
910
914 def __getPhaseSlope(self, metArray, heightList, timeList):
911 def __getPhaseSlope(self, metArray, heightList, timeList):
915 meteorList = []
912 meteorList = []
916 #utctime sec1 height SNR velRad ph0 ph1 ph2 coh0 coh1 coh2
913 #utctime sec1 height SNR velRad ph0 ph1 ph2 coh0 coh1 coh2
917 #Putting back together the meteor matrix
914 #Putting back together the meteor matrix
918 utctime = metArray[:,0]
915 utctime = metArray[:,0]
919 uniqueTime = numpy.unique(utctime)
916 uniqueTime = numpy.unique(utctime)
920
917
921 phaseDerThresh = 0.5
918 phaseDerThresh = 0.5
922 ippSeconds = timeList[1] - timeList[0]
919 ippSeconds = timeList[1] - timeList[0]
923 sec = numpy.where(timeList>1)[0][0]
920 sec = numpy.where(timeList>1)[0][0]
924 nPairs = metArray.shape[1] - 6
921 nPairs = metArray.shape[1] - 6
925 nHeights = len(heightList)
922 nHeights = len(heightList)
926
923
927 for t in uniqueTime:
924 for t in uniqueTime:
928 metArray1 = metArray[utctime==t,:]
925 metArray1 = metArray[utctime==t,:]
929 # phaseDerThresh = numpy.pi/4 #reducir Phase thresh
926 # phaseDerThresh = numpy.pi/4 #reducir Phase thresh
930 tmet = metArray1[:,1].astype(int)
927 tmet = metArray1[:,1].astype(int)
931 hmet = metArray1[:,2].astype(int)
928 hmet = metArray1[:,2].astype(int)
932
929
933 metPhase = numpy.zeros((nPairs, heightList.size, timeList.size - 1))
930 metPhase = numpy.zeros((nPairs, heightList.size, timeList.size - 1))
934 metPhase[:,:] = numpy.nan
931 metPhase[:,:] = numpy.nan
935 metPhase[:,hmet,tmet] = metArray1[:,6:].T
932 metPhase[:,hmet,tmet] = metArray1[:,6:].T
936
933
937 #Delete short trails
934 #Delete short trails
938 metBool = ~numpy.isnan(metPhase[0,:,:])
935 metBool = ~numpy.isnan(metPhase[0,:,:])
939 heightVect = numpy.sum(metBool, axis = 1)
936 heightVect = numpy.sum(metBool, axis = 1)
940 metBool[heightVect<sec,:] = False
937 metBool[heightVect<sec,:] = False
941 metPhase[:,heightVect<sec,:] = numpy.nan
938 metPhase[:,heightVect<sec,:] = numpy.nan
942
939
943 #Derivative
940 #Derivative
944 metDer = numpy.abs(metPhase[:,:,1:] - metPhase[:,:,:-1])
941 metDer = numpy.abs(metPhase[:,:,1:] - metPhase[:,:,:-1])
945 phDerAux = numpy.dstack((numpy.full((nPairs,nHeights,1), False, dtype=bool),metDer > phaseDerThresh))
942 phDerAux = numpy.dstack((numpy.full((nPairs,nHeights,1), False, dtype=bool),metDer > phaseDerThresh))
946 metPhase[phDerAux] = numpy.nan
943 metPhase[phDerAux] = numpy.nan
947
944
948 #--------------------------METEOR DETECTION -----------------------------------------
945 #--------------------------METEOR DETECTION -----------------------------------------
949 indMet = numpy.where(numpy.any(metBool,axis=1))[0]
946 indMet = numpy.where(numpy.any(metBool,axis=1))[0]
950
947
951 for p in numpy.arange(nPairs):
948 for p in numpy.arange(nPairs):
952 phase = metPhase[p,:,:]
949 phase = metPhase[p,:,:]
953 phDer = metDer[p,:,:]
950 phDer = metDer[p,:,:]
954
951
955 for h in indMet:
952 for h in indMet:
956 height = heightList[h]
953 height = heightList[h]
957 phase1 = phase[h,:] #82
954 phase1 = phase[h,:] #82
958 phDer1 = phDer[h,:]
955 phDer1 = phDer[h,:]
959
956
960 phase1[~numpy.isnan(phase1)] = numpy.unwrap(phase1[~numpy.isnan(phase1)]) #Unwrap
957 phase1[~numpy.isnan(phase1)] = numpy.unwrap(phase1[~numpy.isnan(phase1)]) #Unwrap
961
958
962 indValid = numpy.where(~numpy.isnan(phase1))[0]
959 indValid = numpy.where(~numpy.isnan(phase1))[0]
963 initMet = indValid[0]
960 initMet = indValid[0]
964 endMet = 0
961 endMet = 0
965
962
966 for i in range(len(indValid)-1):
963 for i in range(len(indValid)-1):
967
964
968 #Time difference
965 #Time difference
969 inow = indValid[i]
966 inow = indValid[i]
970 inext = indValid[i+1]
967 inext = indValid[i+1]
971 idiff = inext - inow
968 idiff = inext - inow
972 #Phase difference
969 #Phase difference
973 phDiff = numpy.abs(phase1[inext] - phase1[inow])
970 phDiff = numpy.abs(phase1[inext] - phase1[inow])
974
971
975 if idiff>sec or phDiff>numpy.pi/4 or inext==indValid[-1]: #End of Meteor
972 if idiff>sec or phDiff>numpy.pi/4 or inext==indValid[-1]: #End of Meteor
976 sizeTrail = inow - initMet + 1
973 sizeTrail = inow - initMet + 1
977 if sizeTrail>3*sec: #Too short meteors
974 if sizeTrail>3*sec: #Too short meteors
978 x = numpy.arange(initMet,inow+1)*ippSeconds
975 x = numpy.arange(initMet,inow+1)*ippSeconds
979 y = phase1[initMet:inow+1]
976 y = phase1[initMet:inow+1]
980 ynnan = ~numpy.isnan(y)
977 ynnan = ~numpy.isnan(y)
981 x = x[ynnan]
978 x = x[ynnan]
982 y = y[ynnan]
979 y = y[ynnan]
983 slope, intercept, r_value, p_value, std_err = stats.linregress(x,y)
980 slope, intercept, r_value, p_value, std_err = stats.linregress(x,y)
984 ylin = x*slope + intercept
981 ylin = x*slope + intercept
985 rsq = r_value**2
982 rsq = r_value**2
986 if rsq > 0.5:
983 if rsq > 0.5:
987 vel = slope#*height*1000/(k*d)
984 vel = slope#*height*1000/(k*d)
988 estAux = numpy.array([utctime,p,height, vel, rsq])
985 estAux = numpy.array([utctime,p,height, vel, rsq])
989 meteorList.append(estAux)
986 meteorList.append(estAux)
990 initMet = inext
987 initMet = inext
991 metArray2 = numpy.array(meteorList)
988 metArray2 = numpy.array(meteorList)
992
989
993 return metArray2
990 return metArray2
994
991
995 def __calculateAzimuth1(self, rx_location, pairslist, azimuth0):
992 def __calculateAzimuth1(self, rx_location, pairslist, azimuth0):
996
993
997 azimuth1 = numpy.zeros(len(pairslist))
994 azimuth1 = numpy.zeros(len(pairslist))
998 dist = numpy.zeros(len(pairslist))
995 dist = numpy.zeros(len(pairslist))
999
996
1000 for i in range(len(rx_location)):
997 for i in range(len(rx_location)):
1001 ch0 = pairslist[i][0]
998 ch0 = pairslist[i][0]
1002 ch1 = pairslist[i][1]
999 ch1 = pairslist[i][1]
1003
1000
1004 diffX = rx_location[ch0][0] - rx_location[ch1][0]
1001 diffX = rx_location[ch0][0] - rx_location[ch1][0]
1005 diffY = rx_location[ch0][1] - rx_location[ch1][1]
1002 diffY = rx_location[ch0][1] - rx_location[ch1][1]
1006 azimuth1[i] = numpy.arctan2(diffY,diffX)*180/numpy.pi
1003 azimuth1[i] = numpy.arctan2(diffY,diffX)*180/numpy.pi
1007 dist[i] = numpy.sqrt(diffX**2 + diffY**2)
1004 dist[i] = numpy.sqrt(diffX**2 + diffY**2)
1008
1005
1009 azimuth1 -= azimuth0
1006 azimuth1 -= azimuth0
1010 return azimuth1, dist
1007 return azimuth1, dist
1011
1008
1012 def techniqueNSM_DBS(self, **kwargs):
1009 def techniqueNSM_DBS(self, **kwargs):
1013 metArray = kwargs['metArray']
1010 metArray = kwargs['metArray']
1014 heightList = kwargs['heightList']
1011 heightList = kwargs['heightList']
1015 timeList = kwargs['timeList']
1012 timeList = kwargs['timeList']
1016 zenithList = kwargs['zenithList']
1013 zenithList = kwargs['zenithList']
1017 nChan = numpy.max(cmet) + 1
1014 nChan = numpy.max(cmet) + 1
1018 nHeights = len(heightList)
1015 nHeights = len(heightList)
1019
1016
1020 utctime = metArray[:,0]
1017 utctime = metArray[:,0]
1021 cmet = metArray[:,1]
1018 cmet = metArray[:,1]
1022 hmet = metArray1[:,3].astype(int)
1019 hmet = metArray1[:,3].astype(int)
1023 h1met = heightList[hmet]*zenithList[cmet]
1020 h1met = heightList[hmet]*zenithList[cmet]
1024 vmet = metArray1[:,5]
1021 vmet = metArray1[:,5]
1025
1022
1026 for i in range(nHeights - 1):
1023 for i in range(nHeights - 1):
1027 hmin = heightList[i]
1024 hmin = heightList[i]
1028 hmax = heightList[i + 1]
1025 hmax = heightList[i + 1]
1029
1026
1030 vthisH = vmet[(h1met>=hmin) & (h1met<hmax)]
1027 vthisH = vmet[(h1met>=hmin) & (h1met<hmax)]
1031
1028
1032
1029
1033
1030
1034 return data_output
1031 return data_output
1035
1032
1036 def run(self, dataOut, technique, **kwargs):
1033 def run(self, dataOut, technique, **kwargs):
1037
1034
1038 param = dataOut.data_param
1035 param = dataOut.data_param
1039 if dataOut.abscissaList != None:
1036 if dataOut.abscissaList != None:
1040 absc = dataOut.abscissaList[:-1]
1037 absc = dataOut.abscissaList[:-1]
1041 noise = dataOut.noise
1038 noise = dataOut.noise
1042 heightList = dataOut.heightList
1039 heightList = dataOut.heightList
1043 SNR = dataOut.data_SNR
1040 SNR = dataOut.data_SNR
1044
1041
1045 if technique == 'DBS':
1042 if technique == 'DBS':
1046
1043
1047 kwargs['velRadial'] = param[:,1,:] #Radial velocity
1044 kwargs['velRadial'] = param[:,1,:] #Radial velocity
1048 kwargs['heightList'] = heightList
1045 kwargs['heightList'] = heightList
1049 kwargs['SNR'] = SNR
1046 kwargs['SNR'] = SNR
1050
1047
1051 dataOut.data_output, dataOut.heightList, dataOut.data_SNR = self.techniqueDBS(kwargs) #DBS Function
1048 dataOut.data_output, dataOut.heightList, dataOut.data_SNR = self.techniqueDBS(kwargs) #DBS Function
1052 dataOut.utctimeInit = dataOut.utctime
1049 dataOut.utctimeInit = dataOut.utctime
1053 dataOut.outputInterval = dataOut.paramInterval
1050 dataOut.outputInterval = dataOut.paramInterval
1054
1051
1055 elif technique == 'SA':
1052 elif technique == 'SA':
1056
1053
1057 #Parameters
1054 #Parameters
1058 # position_x = kwargs['positionX']
1055 # position_x = kwargs['positionX']
1059 # position_y = kwargs['positionY']
1056 # position_y = kwargs['positionY']
1060 # azimuth = kwargs['azimuth']
1057 # azimuth = kwargs['azimuth']
1061 #
1058 #
1062 # if kwargs.has_key('crosspairsList'):
1059 # if kwargs.has_key('crosspairsList'):
1063 # pairs = kwargs['crosspairsList']
1060 # pairs = kwargs['crosspairsList']
1064 # else:
1061 # else:
1065 # pairs = None
1062 # pairs = None
1066 #
1063 #
1067 # if kwargs.has_key('correctFactor'):
1064 # if kwargs.has_key('correctFactor'):
1068 # correctFactor = kwargs['correctFactor']
1065 # correctFactor = kwargs['correctFactor']
1069 # else:
1066 # else:
1070 # correctFactor = 1
1067 # correctFactor = 1
1071
1068
1072 # tau = dataOut.data_param
1069 # tau = dataOut.data_param
1073 # _lambda = dataOut.C/dataOut.frequency
1070 # _lambda = dataOut.C/dataOut.frequency
1074 # pairsList = dataOut.groupList
1071 # pairsList = dataOut.groupList
1075 # nChannels = dataOut.nChannels
1072 # nChannels = dataOut.nChannels
1076
1073
1077 kwargs['groupList'] = dataOut.groupList
1074 kwargs['groupList'] = dataOut.groupList
1078 kwargs['tau'] = dataOut.data_param
1075 kwargs['tau'] = dataOut.data_param
1079 kwargs['_lambda'] = dataOut.C/dataOut.frequency
1076 kwargs['_lambda'] = dataOut.C/dataOut.frequency
1080 # dataOut.data_output = self.techniqueSA(pairs, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, absc, correctFactor)
1077 # dataOut.data_output = self.techniqueSA(pairs, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, absc, correctFactor)
1081 dataOut.data_output = self.techniqueSA(kwargs)
1078 dataOut.data_output = self.techniqueSA(kwargs)
1082 dataOut.utctimeInit = dataOut.utctime
1079 dataOut.utctimeInit = dataOut.utctime
1083 dataOut.outputInterval = dataOut.timeInterval
1080 dataOut.outputInterval = dataOut.timeInterval
1084
1081
1085 elif technique == 'Meteors':
1082 elif technique == 'Meteors':
1086 dataOut.flagNoData = True
1083 dataOut.flagNoData = True
1087 self.__dataReady = False
1084 self.__dataReady = False
1088
1085
1089 if kwargs.has_key('nHours'):
1086 if kwargs.has_key('nHours'):
1090 nHours = kwargs['nHours']
1087 nHours = kwargs['nHours']
1091 else:
1088 else:
1092 nHours = 1
1089 nHours = 1
1093
1090
1094 if kwargs.has_key('meteorsPerBin'):
1091 if kwargs.has_key('meteorsPerBin'):
1095 meteorThresh = kwargs['meteorsPerBin']
1092 meteorThresh = kwargs['meteorsPerBin']
1096 else:
1093 else:
1097 meteorThresh = 6
1094 meteorThresh = 6
1098
1095
1099 if kwargs.has_key('hmin'):
1096 if kwargs.has_key('hmin'):
1100 hmin = kwargs['hmin']
1097 hmin = kwargs['hmin']
1101 else: hmin = 70
1098 else: hmin = 70
1102 if kwargs.has_key('hmax'):
1099 if kwargs.has_key('hmax'):
1103 hmax = kwargs['hmax']
1100 hmax = kwargs['hmax']
1104 else: hmax = 110
1101 else: hmax = 110
1105
1102
1106 if kwargs.has_key('BinKm'):
1103 if kwargs.has_key('BinKm'):
1107 binkm = kwargs['BinKm']
1104 binkm = kwargs['BinKm']
1108 else:
1105 else:
1109 binkm = 2
1106 binkm = 2
1110
1107
1111 dataOut.outputInterval = nHours*3600
1108 dataOut.outputInterval = nHours*3600
1112
1109
1113 if self.__isConfig == False:
1110 if self.__isConfig == False:
1114 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
1111 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
1115 #Get Initial LTC time
1112 #Get Initial LTC time
1116 self.__initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
1113 self.__initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
1117 self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
1114 self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
1118
1115
1119 self.__isConfig = True
1116 self.__isConfig = True
1120
1117
1121 if self.__buffer is None:
1118 if self.__buffer is None:
1122 self.__buffer = dataOut.data_param
1119 self.__buffer = dataOut.data_param
1123 self.__firstdata = copy.copy(dataOut)
1120 self.__firstdata = copy.copy(dataOut)
1124
1121
1125 else:
1122 else:
1126 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
1123 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
1127
1124
1128 self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
1125 self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
1129
1126
1130 if self.__dataReady:
1127 if self.__dataReady:
1131 dataOut.utctimeInit = self.__initime
1128 dataOut.utctimeInit = self.__initime
1132
1129
1133 self.__initime += dataOut.outputInterval #to erase time offset
1130 self.__initime += dataOut.outputInterval #to erase time offset
1134
1131
1135 dataOut.data_output, dataOut.heightList = self.techniqueMeteors(self.__buffer, meteorThresh, hmin, hmax, binkm)
1132 dataOut.data_output, dataOut.heightList = self.techniqueMeteors(self.__buffer, meteorThresh, hmin, hmax, binkm)
1136 dataOut.flagNoData = False
1133 dataOut.flagNoData = False
1137 self.__buffer = None
1134 self.__buffer = None
1138
1135
1139 elif technique == 'Meteors1':
1136 elif technique == 'Meteors1':
1140 dataOut.flagNoData = True
1137 dataOut.flagNoData = True
1141 self.__dataReady = False
1138 self.__dataReady = False
1142
1139
1143 if kwargs.has_key('nMins'):
1140 if kwargs.has_key('nMins'):
1144 nMins = kwargs['nMins']
1141 nMins = kwargs['nMins']
1145 else: nMins = 20
1142 else: nMins = 20
1146 if kwargs.has_key('rx_location'):
1143 if kwargs.has_key('rx_location'):
1147 rx_location = kwargs['rx_location']
1144 rx_location = kwargs['rx_location']
1148 else: rx_location = [(0,1),(1,1),(1,0)]
1145 else: rx_location = [(0,1),(1,1),(1,0)]
1149 if kwargs.has_key('azimuth'):
1146 if kwargs.has_key('azimuth'):
1150 azimuth = kwargs['azimuth']
1147 azimuth = kwargs['azimuth']
1151 else: azimuth = 51
1148 else: azimuth = 51
1152 if kwargs.has_key('dfactor'):
1149 if kwargs.has_key('dfactor'):
1153 dfactor = kwargs['dfactor']
1150 dfactor = kwargs['dfactor']
1154 if kwargs.has_key('mode'):
1151 if kwargs.has_key('mode'):
1155 mode = kwargs['mode']
1152 mode = kwargs['mode']
1156 else: mode = 'SA'
1153 else: mode = 'SA'
1157
1154
1158 #Borrar luego esto
1155 #Borrar luego esto
1159 if dataOut.groupList is None:
1156 if dataOut.groupList is None:
1160 dataOut.groupList = [(0,1),(0,2),(1,2)]
1157 dataOut.groupList = [(0,1),(0,2),(1,2)]
1161 groupList = dataOut.groupList
1158 groupList = dataOut.groupList
1162 C = 3e8
1159 C = 3e8
1163 freq = 50e6
1160 freq = 50e6
1164 lamb = C/freq
1161 lamb = C/freq
1165 k = 2*numpy.pi/lamb
1162 k = 2*numpy.pi/lamb
1166
1163
1167 timeList = dataOut.abscissaList
1164 timeList = dataOut.abscissaList
1168 heightList = dataOut.heightList
1165 heightList = dataOut.heightList
1169
1166
1170 if self.__isConfig == False:
1167 if self.__isConfig == False:
1171 dataOut.outputInterval = nMins*60
1168 dataOut.outputInterval = nMins*60
1172 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
1169 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
1173 #Get Initial LTC time
1170 #Get Initial LTC time
1174 initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
1171 initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
1175 minuteAux = initime.minute
1172 minuteAux = initime.minute
1176 minuteNew = int(numpy.floor(minuteAux/nMins)*nMins)
1173 minuteNew = int(numpy.floor(minuteAux/nMins)*nMins)
1177 self.__initime = (initime.replace(minute = minuteNew, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
1174 self.__initime = (initime.replace(minute = minuteNew, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
1178
1175
1179 self.__isConfig = True
1176 self.__isConfig = True
1180
1177
1181 if self.__buffer is None:
1178 if self.__buffer is None:
1182 self.__buffer = dataOut.data_param
1179 self.__buffer = dataOut.data_param
1183 self.__firstdata = copy.copy(dataOut)
1180 self.__firstdata = copy.copy(dataOut)
1184
1181
1185 else:
1182 else:
1186 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
1183 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
1187
1184
1188 self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
1185 self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
1189
1186
1190 if self.__dataReady:
1187 if self.__dataReady:
1191 dataOut.utctimeInit = self.__initime
1188 dataOut.utctimeInit = self.__initime
1192 self.__initime += dataOut.outputInterval #to erase time offset
1189 self.__initime += dataOut.outputInterval #to erase time offset
1193
1190
1194 metArray = self.__buffer
1191 metArray = self.__buffer
1195 if mode == 'SA':
1192 if mode == 'SA':
1196 dataOut.data_output = self.techniqueNSM_SA(rx_location=rx_location, groupList=groupList, azimuth=azimuth, dfactor=dfactor, k=k,metArray=metArray, heightList=heightList,timeList=timeList)
1193 dataOut.data_output = self.techniqueNSM_SA(rx_location=rx_location, groupList=groupList, azimuth=azimuth, dfactor=dfactor, k=k,metArray=metArray, heightList=heightList,timeList=timeList)
1197 elif mode == 'DBS':
1194 elif mode == 'DBS':
1198 dataOut.data_output = self.techniqueNSM_DBS(metArray=metArray,heightList=heightList,timeList=timeList)
1195 dataOut.data_output = self.techniqueNSM_DBS(metArray=metArray,heightList=heightList,timeList=timeList)
1199 dataOut.data_output = dataOut.data_output.T
1196 dataOut.data_output = dataOut.data_output.T
1200 dataOut.flagNoData = False
1197 dataOut.flagNoData = False
1201 self.__buffer = None
1198 self.__buffer = None
1202
1199
1203 return
1200 return
1204
1201
1205 class EWDriftsEstimation(Operation):
1202 class EWDriftsEstimation(Operation):
1206
1203
1207 def __init__(self):
1208 Operation.__init__(self)
1209
1204
1210 def __correctValues(self, heiRang, phi, velRadial, SNR):
1205 def __correctValues(self, heiRang, phi, velRadial, SNR):
1211 listPhi = phi.tolist()
1206 listPhi = phi.tolist()
1212 maxid = listPhi.index(max(listPhi))
1207 maxid = listPhi.index(max(listPhi))
1213 minid = listPhi.index(min(listPhi))
1208 minid = listPhi.index(min(listPhi))
1214
1209
1215 rango = range(len(phi))
1210 rango = range(len(phi))
1216 # rango = numpy.delete(rango,maxid)
1211 # rango = numpy.delete(rango,maxid)
1217
1212
1218 heiRang1 = heiRang*math.cos(phi[maxid])
1213 heiRang1 = heiRang*math.cos(phi[maxid])
1219 heiRangAux = heiRang*math.cos(phi[minid])
1214 heiRangAux = heiRang*math.cos(phi[minid])
1220 indOut = (heiRang1 < heiRangAux[0]).nonzero()
1215 indOut = (heiRang1 < heiRangAux[0]).nonzero()
1221 heiRang1 = numpy.delete(heiRang1,indOut)
1216 heiRang1 = numpy.delete(heiRang1,indOut)
1222
1217
1223 velRadial1 = numpy.zeros([len(phi),len(heiRang1)])
1218 velRadial1 = numpy.zeros([len(phi),len(heiRang1)])
1224 SNR1 = numpy.zeros([len(phi),len(heiRang1)])
1219 SNR1 = numpy.zeros([len(phi),len(heiRang1)])
1225
1220
1226 for i in rango:
1221 for i in rango:
1227 x = heiRang*math.cos(phi[i])
1222 x = heiRang*math.cos(phi[i])
1228 y1 = velRadial[i,:]
1223 y1 = velRadial[i,:]
1229 f1 = interpolate.interp1d(x,y1,kind = 'cubic')
1224 f1 = interpolate.interp1d(x,y1,kind = 'cubic')
1230
1225
1231 x1 = heiRang1
1226 x1 = heiRang1
1232 y11 = f1(x1)
1227 y11 = f1(x1)
1233
1228
1234 y2 = SNR[i,:]
1229 y2 = SNR[i,:]
1235 f2 = interpolate.interp1d(x,y2,kind = 'cubic')
1230 f2 = interpolate.interp1d(x,y2,kind = 'cubic')
1236 y21 = f2(x1)
1231 y21 = f2(x1)
1237
1232
1238 velRadial1[i,:] = y11
1233 velRadial1[i,:] = y11
1239 SNR1[i,:] = y21
1234 SNR1[i,:] = y21
1240
1235
1241 return heiRang1, velRadial1, SNR1
1236 return heiRang1, velRadial1, SNR1
1242
1237
1243 def run(self, dataOut, zenith, zenithCorrection):
1238 def run(self, dataOut, zenith, zenithCorrection):
1244 heiRang = dataOut.heightList
1239 heiRang = dataOut.heightList
1245 velRadial = dataOut.data_param[:,3,:]
1240 velRadial = dataOut.data_param[:,3,:]
1246 SNR = dataOut.data_SNR
1241 SNR = dataOut.data_SNR
1247
1242
1248 zenith = numpy.array(zenith)
1243 zenith = numpy.array(zenith)
1249 zenith -= zenithCorrection
1244 zenith -= zenithCorrection
1250 zenith *= numpy.pi/180
1245 zenith *= numpy.pi/180
1251
1246
1252 heiRang1, velRadial1, SNR1 = self.__correctValues(heiRang, numpy.abs(zenith), velRadial, SNR)
1247 heiRang1, velRadial1, SNR1 = self.__correctValues(heiRang, numpy.abs(zenith), velRadial, SNR)
1253
1248
1254 alp = zenith[0]
1249 alp = zenith[0]
1255 bet = zenith[1]
1250 bet = zenith[1]
1256
1251
1257 w_w = velRadial1[0,:]
1252 w_w = velRadial1[0,:]
1258 w_e = velRadial1[1,:]
1253 w_e = velRadial1[1,:]
1259
1254
1260 w = (w_w*numpy.sin(bet) - w_e*numpy.sin(alp))/(numpy.cos(alp)*numpy.sin(bet) - numpy.cos(bet)*numpy.sin(alp))
1255 w = (w_w*numpy.sin(bet) - w_e*numpy.sin(alp))/(numpy.cos(alp)*numpy.sin(bet) - numpy.cos(bet)*numpy.sin(alp))
1261 u = (w_w*numpy.cos(bet) - w_e*numpy.cos(alp))/(numpy.sin(alp)*numpy.cos(bet) - numpy.sin(bet)*numpy.cos(alp))
1256 u = (w_w*numpy.cos(bet) - w_e*numpy.cos(alp))/(numpy.sin(alp)*numpy.cos(bet) - numpy.sin(bet)*numpy.cos(alp))
1262
1257
1263 winds = numpy.vstack((u,w))
1258 winds = numpy.vstack((u,w))
1264
1259
1265 dataOut.heightList = heiRang1
1260 dataOut.heightList = heiRang1
1266 dataOut.data_output = winds
1261 dataOut.data_output = winds
1267 dataOut.data_SNR = SNR1
1262 dataOut.data_SNR = SNR1
1268
1263
1269 dataOut.utctimeInit = dataOut.utctime
1264 dataOut.utctimeInit = dataOut.utctime
1270 dataOut.outputInterval = dataOut.timeInterval
1265 dataOut.outputInterval = dataOut.timeInterval
1271 return
1266 return
1272
1267
1273 #--------------- Non Specular Meteor ----------------
1268 #--------------- Non Specular Meteor ----------------
1274
1269
1275 class NonSpecularMeteorDetection(Operation):
1270 class NonSpecularMeteorDetection(Operation):
1276
1271
1277 def run(self, mode, SNRthresh=8, phaseDerThresh=0.5, cohThresh=0.8, allData = False):
1272 def run(self, mode, SNRthresh=8, phaseDerThresh=0.5, cohThresh=0.8, allData = False):
1278 data_acf = self.dataOut.data_pre[0]
1273 data_acf = self.dataOut.data_pre[0]
1279 data_ccf = self.dataOut.data_pre[1]
1274 data_ccf = self.dataOut.data_pre[1]
1280
1275
1281 lamb = self.dataOut.C/self.dataOut.frequency
1276 lamb = self.dataOut.C/self.dataOut.frequency
1282 tSamp = self.dataOut.ippSeconds*self.dataOut.nCohInt
1277 tSamp = self.dataOut.ippSeconds*self.dataOut.nCohInt
1283 paramInterval = self.dataOut.paramInterval
1278 paramInterval = self.dataOut.paramInterval
1284
1279
1285 nChannels = data_acf.shape[0]
1280 nChannels = data_acf.shape[0]
1286 nLags = data_acf.shape[1]
1281 nLags = data_acf.shape[1]
1287 nProfiles = data_acf.shape[2]
1282 nProfiles = data_acf.shape[2]
1288 nHeights = self.dataOut.nHeights
1283 nHeights = self.dataOut.nHeights
1289 nCohInt = self.dataOut.nCohInt
1284 nCohInt = self.dataOut.nCohInt
1290 sec = numpy.round(nProfiles/self.dataOut.paramInterval)
1285 sec = numpy.round(nProfiles/self.dataOut.paramInterval)
1291 heightList = self.dataOut.heightList
1286 heightList = self.dataOut.heightList
1292 ippSeconds = self.dataOut.ippSeconds*self.dataOut.nCohInt*self.dataOut.nAvg
1287 ippSeconds = self.dataOut.ippSeconds*self.dataOut.nCohInt*self.dataOut.nAvg
1293 utctime = self.dataOut.utctime
1288 utctime = self.dataOut.utctime
1294
1289
1295 self.dataOut.abscissaList = numpy.arange(0,paramInterval+ippSeconds,ippSeconds)
1290 self.dataOut.abscissaList = numpy.arange(0,paramInterval+ippSeconds,ippSeconds)
1296
1291
1297 #------------------------ SNR --------------------------------------
1292 #------------------------ SNR --------------------------------------
1298 power = data_acf[:,0,:,:].real
1293 power = data_acf[:,0,:,:].real
1299 noise = numpy.zeros(nChannels)
1294 noise = numpy.zeros(nChannels)
1300 SNR = numpy.zeros(power.shape)
1295 SNR = numpy.zeros(power.shape)
1301 for i in range(nChannels):
1296 for i in range(nChannels):
1302 noise[i] = hildebrand_sekhon(power[i,:], nCohInt)
1297 noise[i] = hildebrand_sekhon(power[i,:], nCohInt)
1303 SNR[i] = (power[i]-noise[i])/noise[i]
1298 SNR[i] = (power[i]-noise[i])/noise[i]
1304 SNRm = numpy.nanmean(SNR, axis = 0)
1299 SNRm = numpy.nanmean(SNR, axis = 0)
1305 SNRdB = 10*numpy.log10(SNR)
1300 SNRdB = 10*numpy.log10(SNR)
1306
1301
1307 if mode == 'SA':
1302 if mode == 'SA':
1308 nPairs = data_ccf.shape[0]
1303 nPairs = data_ccf.shape[0]
1309 #---------------------- Coherence and Phase --------------------------
1304 #---------------------- Coherence and Phase --------------------------
1310 phase = numpy.zeros(data_ccf[:,0,:,:].shape)
1305 phase = numpy.zeros(data_ccf[:,0,:,:].shape)
1311 # phase1 = numpy.copy(phase)
1306 # phase1 = numpy.copy(phase)
1312 coh1 = numpy.zeros(data_ccf[:,0,:,:].shape)
1307 coh1 = numpy.zeros(data_ccf[:,0,:,:].shape)
1313
1308
1314 for p in range(nPairs):
1309 for p in range(nPairs):
1315 ch0 = self.dataOut.groupList[p][0]
1310 ch0 = self.dataOut.groupList[p][0]
1316 ch1 = self.dataOut.groupList[p][1]
1311 ch1 = self.dataOut.groupList[p][1]
1317 ccf = data_ccf[p,0,:,:]/numpy.sqrt(data_acf[ch0,0,:,:]*data_acf[ch1,0,:,:])
1312 ccf = data_ccf[p,0,:,:]/numpy.sqrt(data_acf[ch0,0,:,:]*data_acf[ch1,0,:,:])
1318 phase[p,:,:] = ndimage.median_filter(numpy.angle(ccf), size = (5,1)) #median filter
1313 phase[p,:,:] = ndimage.median_filter(numpy.angle(ccf), size = (5,1)) #median filter
1319 # phase1[p,:,:] = numpy.angle(ccf) #median filter
1314 # phase1[p,:,:] = numpy.angle(ccf) #median filter
1320 coh1[p,:,:] = ndimage.median_filter(numpy.abs(ccf), 5) #median filter
1315 coh1[p,:,:] = ndimage.median_filter(numpy.abs(ccf), 5) #median filter
1321 # coh1[p,:,:] = numpy.abs(ccf) #median filter
1316 # coh1[p,:,:] = numpy.abs(ccf) #median filter
1322 coh = numpy.nanmax(coh1, axis = 0)
1317 coh = numpy.nanmax(coh1, axis = 0)
1323 # struc = numpy.ones((5,1))
1318 # struc = numpy.ones((5,1))
1324 # coh = ndimage.morphology.grey_dilation(coh, size=(10,1))
1319 # coh = ndimage.morphology.grey_dilation(coh, size=(10,1))
1325 #---------------------- Radial Velocity ----------------------------
1320 #---------------------- Radial Velocity ----------------------------
1326 phaseAux = numpy.mean(numpy.angle(data_acf[:,1,:,:]), axis = 0)
1321 phaseAux = numpy.mean(numpy.angle(data_acf[:,1,:,:]), axis = 0)
1327 velRad = phaseAux*lamb/(4*numpy.pi*tSamp)
1322 velRad = phaseAux*lamb/(4*numpy.pi*tSamp)
1328
1323
1329 if allData:
1324 if allData:
1330 boolMetFin = ~numpy.isnan(SNRm)
1325 boolMetFin = ~numpy.isnan(SNRm)
1331 # coh[:-1,:] = numpy.nanmean(numpy.abs(phase[:,1:,:] - phase[:,:-1,:]),axis=0)
1326 # coh[:-1,:] = numpy.nanmean(numpy.abs(phase[:,1:,:] - phase[:,:-1,:]),axis=0)
1332 else:
1327 else:
1333 #------------------------ Meteor mask ---------------------------------
1328 #------------------------ Meteor mask ---------------------------------
1334 # #SNR mask
1329 # #SNR mask
1335 # boolMet = (SNRdB>SNRthresh)#|(~numpy.isnan(SNRdB))
1330 # boolMet = (SNRdB>SNRthresh)#|(~numpy.isnan(SNRdB))
1336 #
1331 #
1337 # #Erase small objects
1332 # #Erase small objects
1338 # boolMet1 = self.__erase_small(boolMet, 2*sec, 5)
1333 # boolMet1 = self.__erase_small(boolMet, 2*sec, 5)
1339 #
1334 #
1340 # auxEEJ = numpy.sum(boolMet1,axis=0)
1335 # auxEEJ = numpy.sum(boolMet1,axis=0)
1341 # indOver = auxEEJ>nProfiles*0.8 #Use this later
1336 # indOver = auxEEJ>nProfiles*0.8 #Use this later
1342 # indEEJ = numpy.where(indOver)[0]
1337 # indEEJ = numpy.where(indOver)[0]
1343 # indNEEJ = numpy.where(~indOver)[0]
1338 # indNEEJ = numpy.where(~indOver)[0]
1344 #
1339 #
1345 # boolMetFin = boolMet1
1340 # boolMetFin = boolMet1
1346 #
1341 #
1347 # if indEEJ.size > 0:
1342 # if indEEJ.size > 0:
1348 # boolMet1[:,indEEJ] = False #Erase heights with EEJ
1343 # boolMet1[:,indEEJ] = False #Erase heights with EEJ
1349 #
1344 #
1350 # boolMet2 = coh > cohThresh
1345 # boolMet2 = coh > cohThresh
1351 # boolMet2 = self.__erase_small(boolMet2, 2*sec,5)
1346 # boolMet2 = self.__erase_small(boolMet2, 2*sec,5)
1352 #
1347 #
1353 # #Final Meteor mask
1348 # #Final Meteor mask
1354 # boolMetFin = boolMet1|boolMet2
1349 # boolMetFin = boolMet1|boolMet2
1355
1350
1356 #Coherence mask
1351 #Coherence mask
1357 boolMet1 = coh > 0.75
1352 boolMet1 = coh > 0.75
1358 struc = numpy.ones((30,1))
1353 struc = numpy.ones((30,1))
1359 boolMet1 = ndimage.morphology.binary_dilation(boolMet1, structure=struc)
1354 boolMet1 = ndimage.morphology.binary_dilation(boolMet1, structure=struc)
1360
1355
1361 #Derivative mask
1356 #Derivative mask
1362 derPhase = numpy.nanmean(numpy.abs(phase[:,1:,:] - phase[:,:-1,:]),axis=0)
1357 derPhase = numpy.nanmean(numpy.abs(phase[:,1:,:] - phase[:,:-1,:]),axis=0)
1363 boolMet2 = derPhase < 0.2
1358 boolMet2 = derPhase < 0.2
1364 # boolMet2 = ndimage.morphology.binary_opening(boolMet2)
1359 # boolMet2 = ndimage.morphology.binary_opening(boolMet2)
1365 # boolMet2 = ndimage.morphology.binary_closing(boolMet2, structure = numpy.ones((10,1)))
1360 # boolMet2 = ndimage.morphology.binary_closing(boolMet2, structure = numpy.ones((10,1)))
1366 boolMet2 = ndimage.median_filter(boolMet2,size=5)
1361 boolMet2 = ndimage.median_filter(boolMet2,size=5)
1367 boolMet2 = numpy.vstack((boolMet2,numpy.full((1,nHeights), True, dtype=bool)))
1362 boolMet2 = numpy.vstack((boolMet2,numpy.full((1,nHeights), True, dtype=bool)))
1368 # #Final mask
1363 # #Final mask
1369 # boolMetFin = boolMet2
1364 # boolMetFin = boolMet2
1370 boolMetFin = boolMet1&boolMet2
1365 boolMetFin = boolMet1&boolMet2
1371 # boolMetFin = ndimage.morphology.binary_dilation(boolMetFin)
1366 # boolMetFin = ndimage.morphology.binary_dilation(boolMetFin)
1372 #Creating data_param
1367 #Creating data_param
1373 coordMet = numpy.where(boolMetFin)
1368 coordMet = numpy.where(boolMetFin)
1374
1369
1375 tmet = coordMet[0]
1370 tmet = coordMet[0]
1376 hmet = coordMet[1]
1371 hmet = coordMet[1]
1377
1372
1378 data_param = numpy.zeros((tmet.size, 6 + nPairs))
1373 data_param = numpy.zeros((tmet.size, 6 + nPairs))
1379 data_param[:,0] = utctime
1374 data_param[:,0] = utctime
1380 data_param[:,1] = tmet
1375 data_param[:,1] = tmet
1381 data_param[:,2] = hmet
1376 data_param[:,2] = hmet
1382 data_param[:,3] = SNRm[tmet,hmet]
1377 data_param[:,3] = SNRm[tmet,hmet]
1383 data_param[:,4] = velRad[tmet,hmet]
1378 data_param[:,4] = velRad[tmet,hmet]
1384 data_param[:,5] = coh[tmet,hmet]
1379 data_param[:,5] = coh[tmet,hmet]
1385 data_param[:,6:] = phase[:,tmet,hmet].T
1380 data_param[:,6:] = phase[:,tmet,hmet].T
1386
1381
1387 elif mode == 'DBS':
1382 elif mode == 'DBS':
1388 self.dataOut.groupList = numpy.arange(nChannels)
1383 self.dataOut.groupList = numpy.arange(nChannels)
1389
1384
1390 #Radial Velocities
1385 #Radial Velocities
1391 # phase = numpy.angle(data_acf[:,1,:,:])
1386 # phase = numpy.angle(data_acf[:,1,:,:])
1392 phase = ndimage.median_filter(numpy.angle(data_acf[:,1,:,:]), size = (1,5,1))
1387 phase = ndimage.median_filter(numpy.angle(data_acf[:,1,:,:]), size = (1,5,1))
1393 velRad = phase*lamb/(4*numpy.pi*tSamp)
1388 velRad = phase*lamb/(4*numpy.pi*tSamp)
1394
1389
1395 #Spectral width
1390 #Spectral width
1396 acf1 = ndimage.median_filter(numpy.abs(data_acf[:,1,:,:]), size = (1,5,1))
1391 acf1 = ndimage.median_filter(numpy.abs(data_acf[:,1,:,:]), size = (1,5,1))
1397 acf2 = ndimage.median_filter(numpy.abs(data_acf[:,2,:,:]), size = (1,5,1))
1392 acf2 = ndimage.median_filter(numpy.abs(data_acf[:,2,:,:]), size = (1,5,1))
1398
1393
1399 spcWidth = (lamb/(2*numpy.sqrt(6)*numpy.pi*tSamp))*numpy.sqrt(numpy.log(acf1/acf2))
1394 spcWidth = (lamb/(2*numpy.sqrt(6)*numpy.pi*tSamp))*numpy.sqrt(numpy.log(acf1/acf2))
1400 # velRad = ndimage.median_filter(velRad, size = (1,5,1))
1395 # velRad = ndimage.median_filter(velRad, size = (1,5,1))
1401 if allData:
1396 if allData:
1402 boolMetFin = ~numpy.isnan(SNRdB)
1397 boolMetFin = ~numpy.isnan(SNRdB)
1403 else:
1398 else:
1404 #SNR
1399 #SNR
1405 boolMet1 = (SNRdB>SNRthresh) #SNR mask
1400 boolMet1 = (SNRdB>SNRthresh) #SNR mask
1406 boolMet1 = ndimage.median_filter(boolMet1, size=(1,5,5))
1401 boolMet1 = ndimage.median_filter(boolMet1, size=(1,5,5))
1407
1402
1408 #Radial velocity
1403 #Radial velocity
1409 boolMet2 = numpy.abs(velRad) < 30
1404 boolMet2 = numpy.abs(velRad) < 30
1410 boolMet2 = ndimage.median_filter(boolMet2, (1,5,5))
1405 boolMet2 = ndimage.median_filter(boolMet2, (1,5,5))
1411
1406
1412 #Spectral Width
1407 #Spectral Width
1413 boolMet3 = spcWidth < 30
1408 boolMet3 = spcWidth < 30
1414 boolMet3 = ndimage.median_filter(boolMet3, (1,5,5))
1409 boolMet3 = ndimage.median_filter(boolMet3, (1,5,5))
1415 # boolMetFin = self.__erase_small(boolMet1, 10,5)
1410 # boolMetFin = self.__erase_small(boolMet1, 10,5)
1416 boolMetFin = boolMet1&boolMet2&boolMet3
1411 boolMetFin = boolMet1&boolMet2&boolMet3
1417
1412
1418 #Creating data_param
1413 #Creating data_param
1419 coordMet = numpy.where(boolMetFin)
1414 coordMet = numpy.where(boolMetFin)
1420
1415
1421 cmet = coordMet[0]
1416 cmet = coordMet[0]
1422 tmet = coordMet[1]
1417 tmet = coordMet[1]
1423 hmet = coordMet[2]
1418 hmet = coordMet[2]
1424
1419
1425 data_param = numpy.zeros((tmet.size, 7))
1420 data_param = numpy.zeros((tmet.size, 7))
1426 data_param[:,0] = utctime
1421 data_param[:,0] = utctime
1427 data_param[:,1] = cmet
1422 data_param[:,1] = cmet
1428 data_param[:,2] = tmet
1423 data_param[:,2] = tmet
1429 data_param[:,3] = hmet
1424 data_param[:,3] = hmet
1430 data_param[:,4] = SNR[cmet,tmet,hmet].T
1425 data_param[:,4] = SNR[cmet,tmet,hmet].T
1431 data_param[:,5] = velRad[cmet,tmet,hmet].T
1426 data_param[:,5] = velRad[cmet,tmet,hmet].T
1432 data_param[:,6] = spcWidth[cmet,tmet,hmet].T
1427 data_param[:,6] = spcWidth[cmet,tmet,hmet].T
1433
1428
1434 # self.dataOut.data_param = data_int
1429 # self.dataOut.data_param = data_int
1435 if len(data_param) == 0:
1430 if len(data_param) == 0:
1436 self.dataOut.flagNoData = True
1431 self.dataOut.flagNoData = True
1437 else:
1432 else:
1438 self.dataOut.data_param = data_param
1433 self.dataOut.data_param = data_param
1439
1434
1440 def __erase_small(self, binArray, threshX, threshY):
1435 def __erase_small(self, binArray, threshX, threshY):
1441 labarray, numfeat = ndimage.measurements.label(binArray)
1436 labarray, numfeat = ndimage.measurements.label(binArray)
1442 binArray1 = numpy.copy(binArray)
1437 binArray1 = numpy.copy(binArray)
1443
1438
1444 for i in range(1,numfeat + 1):
1439 for i in range(1,numfeat + 1):
1445 auxBin = (labarray==i)
1440 auxBin = (labarray==i)
1446 auxSize = auxBin.sum()
1441 auxSize = auxBin.sum()
1447
1442
1448 x,y = numpy.where(auxBin)
1443 x,y = numpy.where(auxBin)
1449 widthX = x.max() - x.min()
1444 widthX = x.max() - x.min()
1450 widthY = y.max() - y.min()
1445 widthY = y.max() - y.min()
1451
1446
1452 #width X: 3 seg -> 12.5*3
1447 #width X: 3 seg -> 12.5*3
1453 #width Y:
1448 #width Y:
1454
1449
1455 if (auxSize < 50) or (widthX < threshX) or (widthY < threshY):
1450 if (auxSize < 50) or (widthX < threshX) or (widthY < threshY):
1456 binArray1[auxBin] = False
1451 binArray1[auxBin] = False
1457
1452
1458 return binArray1
1453 return binArray1
1459
1454
1460 #--------------- Specular Meteor ----------------
1455 #--------------- Specular Meteor ----------------
1461
1456
1462 class SMDetection(Operation):
1457 class SMDetection(Operation):
1463 '''
1458 '''
1464 Function DetectMeteors()
1459 Function DetectMeteors()
1465 Project developed with paper:
1460 Project developed with paper:
1466 HOLDSWORTH ET AL. 2004
1461 HOLDSWORTH ET AL. 2004
1467
1462
1468 Input:
1463 Input:
1469 self.dataOut.data_pre
1464 self.dataOut.data_pre
1470
1465
1471 centerReceiverIndex: From the channels, which is the center receiver
1466 centerReceiverIndex: From the channels, which is the center receiver
1472
1467
1473 hei_ref: Height reference for the Beacon signal extraction
1468 hei_ref: Height reference for the Beacon signal extraction
1474 tauindex:
1469 tauindex:
1475 predefinedPhaseShifts: Predefined phase offset for the voltge signals
1470 predefinedPhaseShifts: Predefined phase offset for the voltge signals
1476
1471
1477 cohDetection: Whether to user Coherent detection or not
1472 cohDetection: Whether to user Coherent detection or not
1478 cohDet_timeStep: Coherent Detection calculation time step
1473 cohDet_timeStep: Coherent Detection calculation time step
1479 cohDet_thresh: Coherent Detection phase threshold to correct phases
1474 cohDet_thresh: Coherent Detection phase threshold to correct phases
1480
1475
1481 noise_timeStep: Noise calculation time step
1476 noise_timeStep: Noise calculation time step
1482 noise_multiple: Noise multiple to define signal threshold
1477 noise_multiple: Noise multiple to define signal threshold
1483
1478
1484 multDet_timeLimit: Multiple Detection Removal time limit in seconds
1479 multDet_timeLimit: Multiple Detection Removal time limit in seconds
1485 multDet_rangeLimit: Multiple Detection Removal range limit in km
1480 multDet_rangeLimit: Multiple Detection Removal range limit in km
1486
1481
1487 phaseThresh: Maximum phase difference between receiver to be consider a meteor
1482 phaseThresh: Maximum phase difference between receiver to be consider a meteor
1488 SNRThresh: Minimum SNR threshold of the meteor signal to be consider a meteor
1483 SNRThresh: Minimum SNR threshold of the meteor signal to be consider a meteor
1489
1484
1490 hmin: Minimum Height of the meteor to use it in the further wind estimations
1485 hmin: Minimum Height of the meteor to use it in the further wind estimations
1491 hmax: Maximum Height of the meteor to use it in the further wind estimations
1486 hmax: Maximum Height of the meteor to use it in the further wind estimations
1492 azimuth: Azimuth angle correction
1487 azimuth: Azimuth angle correction
1493
1488
1494 Affected:
1489 Affected:
1495 self.dataOut.data_param
1490 self.dataOut.data_param
1496
1491
1497 Rejection Criteria (Errors):
1492 Rejection Criteria (Errors):
1498 0: No error; analysis OK
1493 0: No error; analysis OK
1499 1: SNR < SNR threshold
1494 1: SNR < SNR threshold
1500 2: angle of arrival (AOA) ambiguously determined
1495 2: angle of arrival (AOA) ambiguously determined
1501 3: AOA estimate not feasible
1496 3: AOA estimate not feasible
1502 4: Large difference in AOAs obtained from different antenna baselines
1497 4: Large difference in AOAs obtained from different antenna baselines
1503 5: echo at start or end of time series
1498 5: echo at start or end of time series
1504 6: echo less than 5 examples long; too short for analysis
1499 6: echo less than 5 examples long; too short for analysis
1505 7: echo rise exceeds 0.3s
1500 7: echo rise exceeds 0.3s
1506 8: echo decay time less than twice rise time
1501 8: echo decay time less than twice rise time
1507 9: large power level before echo
1502 9: large power level before echo
1508 10: large power level after echo
1503 10: large power level after echo
1509 11: poor fit to amplitude for estimation of decay time
1504 11: poor fit to amplitude for estimation of decay time
1510 12: poor fit to CCF phase variation for estimation of radial drift velocity
1505 12: poor fit to CCF phase variation for estimation of radial drift velocity
1511 13: height unresolvable echo: not valid height within 70 to 110 km
1506 13: height unresolvable echo: not valid height within 70 to 110 km
1512 14: height ambiguous echo: more then one possible height within 70 to 110 km
1507 14: height ambiguous echo: more then one possible height within 70 to 110 km
1513 15: radial drift velocity or projected horizontal velocity exceeds 200 m/s
1508 15: radial drift velocity or projected horizontal velocity exceeds 200 m/s
1514 16: oscilatory echo, indicating event most likely not an underdense echo
1509 16: oscilatory echo, indicating event most likely not an underdense echo
1515
1510
1516 17: phase difference in meteor Reestimation
1511 17: phase difference in meteor Reestimation
1517
1512
1518 Data Storage:
1513 Data Storage:
1519 Meteors for Wind Estimation (8):
1514 Meteors for Wind Estimation (8):
1520 Utc Time | Range Height
1515 Utc Time | Range Height
1521 Azimuth Zenith errorCosDir
1516 Azimuth Zenith errorCosDir
1522 VelRad errorVelRad
1517 VelRad errorVelRad
1523 Phase0 Phase1 Phase2 Phase3
1518 Phase0 Phase1 Phase2 Phase3
1524 TypeError
1519 TypeError
1525
1520
1526 '''
1521 '''
1527
1522
1528 def run(self, dataOut, hei_ref = None, tauindex = 0,
1523 def run(self, dataOut, hei_ref = None, tauindex = 0,
1529 phaseOffsets = None,
1524 phaseOffsets = None,
1530 cohDetection = False, cohDet_timeStep = 1, cohDet_thresh = 25,
1525 cohDetection = False, cohDet_timeStep = 1, cohDet_thresh = 25,
1531 noise_timeStep = 4, noise_multiple = 4,
1526 noise_timeStep = 4, noise_multiple = 4,
1532 multDet_timeLimit = 1, multDet_rangeLimit = 3,
1527 multDet_timeLimit = 1, multDet_rangeLimit = 3,
1533 phaseThresh = 20, SNRThresh = 5,
1528 phaseThresh = 20, SNRThresh = 5,
1534 hmin = 50, hmax=150, azimuth = 0,
1529 hmin = 50, hmax=150, azimuth = 0,
1535 channelPositions = None) :
1530 channelPositions = None) :
1536
1531
1537
1532
1538 #Getting Pairslist
1533 #Getting Pairslist
1539 if channelPositions is None:
1534 if channelPositions is None:
1540 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
1535 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
1541 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
1536 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
1542 meteorOps = SMOperations()
1537 meteorOps = SMOperations()
1543 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
1538 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
1544 heiRang = dataOut.getHeiRange()
1539 heiRang = dataOut.getHeiRange()
1545 #Get Beacon signal - No Beacon signal anymore
1540 #Get Beacon signal - No Beacon signal anymore
1546 # newheis = numpy.where(self.dataOut.heightList>self.dataOut.radarControllerHeaderObj.Taus[tauindex])
1541 # newheis = numpy.where(self.dataOut.heightList>self.dataOut.radarControllerHeaderObj.Taus[tauindex])
1547 #
1542 #
1548 # if hei_ref != None:
1543 # if hei_ref != None:
1549 # newheis = numpy.where(self.dataOut.heightList>hei_ref)
1544 # newheis = numpy.where(self.dataOut.heightList>hei_ref)
1550 #
1545 #
1551
1546
1552
1547
1553 #****************REMOVING HARDWARE PHASE DIFFERENCES***************
1548 #****************REMOVING HARDWARE PHASE DIFFERENCES***************
1554 # see if the user put in pre defined phase shifts
1549 # see if the user put in pre defined phase shifts
1555 voltsPShift = dataOut.data_pre.copy()
1550 voltsPShift = dataOut.data_pre.copy()
1556
1551
1557 # if predefinedPhaseShifts != None:
1552 # if predefinedPhaseShifts != None:
1558 # hardwarePhaseShifts = numpy.array(predefinedPhaseShifts)*numpy.pi/180
1553 # hardwarePhaseShifts = numpy.array(predefinedPhaseShifts)*numpy.pi/180
1559 #
1554 #
1560 # # elif beaconPhaseShifts:
1555 # # elif beaconPhaseShifts:
1561 # # #get hardware phase shifts using beacon signal
1556 # # #get hardware phase shifts using beacon signal
1562 # # hardwarePhaseShifts = self.__getHardwarePhaseDiff(self.dataOut.data_pre, pairslist, newheis, 10)
1557 # # hardwarePhaseShifts = self.__getHardwarePhaseDiff(self.dataOut.data_pre, pairslist, newheis, 10)
1563 # # hardwarePhaseShifts = numpy.insert(hardwarePhaseShifts,centerReceiverIndex,0)
1558 # # hardwarePhaseShifts = numpy.insert(hardwarePhaseShifts,centerReceiverIndex,0)
1564 #
1559 #
1565 # else:
1560 # else:
1566 # hardwarePhaseShifts = numpy.zeros(5)
1561 # hardwarePhaseShifts = numpy.zeros(5)
1567 #
1562 #
1568 # voltsPShift = numpy.zeros((self.dataOut.data_pre.shape[0],self.dataOut.data_pre.shape[1],self.dataOut.data_pre.shape[2]), dtype = 'complex')
1563 # voltsPShift = numpy.zeros((self.dataOut.data_pre.shape[0],self.dataOut.data_pre.shape[1],self.dataOut.data_pre.shape[2]), dtype = 'complex')
1569 # for i in range(self.dataOut.data_pre.shape[0]):
1564 # for i in range(self.dataOut.data_pre.shape[0]):
1570 # voltsPShift[i,:,:] = self.__shiftPhase(self.dataOut.data_pre[i,:,:], hardwarePhaseShifts[i])
1565 # voltsPShift[i,:,:] = self.__shiftPhase(self.dataOut.data_pre[i,:,:], hardwarePhaseShifts[i])
1571
1566
1572 #******************END OF REMOVING HARDWARE PHASE DIFFERENCES*********
1567 #******************END OF REMOVING HARDWARE PHASE DIFFERENCES*********
1573
1568
1574 #Remove DC
1569 #Remove DC
1575 voltsDC = numpy.mean(voltsPShift,1)
1570 voltsDC = numpy.mean(voltsPShift,1)
1576 voltsDC = numpy.mean(voltsDC,1)
1571 voltsDC = numpy.mean(voltsDC,1)
1577 for i in range(voltsDC.shape[0]):
1572 for i in range(voltsDC.shape[0]):
1578 voltsPShift[i] = voltsPShift[i] - voltsDC[i]
1573 voltsPShift[i] = voltsPShift[i] - voltsDC[i]
1579
1574
1580 #Don't considerate last heights, theyre used to calculate Hardware Phase Shift
1575 #Don't considerate last heights, theyre used to calculate Hardware Phase Shift
1581 # voltsPShift = voltsPShift[:,:,:newheis[0][0]]
1576 # voltsPShift = voltsPShift[:,:,:newheis[0][0]]
1582
1577
1583 #************ FIND POWER OF DATA W/COH OR NON COH DETECTION (3.4) **********
1578 #************ FIND POWER OF DATA W/COH OR NON COH DETECTION (3.4) **********
1584 #Coherent Detection
1579 #Coherent Detection
1585 if cohDetection:
1580 if cohDetection:
1586 #use coherent detection to get the net power
1581 #use coherent detection to get the net power
1587 cohDet_thresh = cohDet_thresh*numpy.pi/180
1582 cohDet_thresh = cohDet_thresh*numpy.pi/180
1588 voltsPShift = self.__coherentDetection(voltsPShift, cohDet_timeStep, dataOut.timeInterval, pairslist0, cohDet_thresh)
1583 voltsPShift = self.__coherentDetection(voltsPShift, cohDet_timeStep, dataOut.timeInterval, pairslist0, cohDet_thresh)
1589
1584
1590 #Non-coherent detection!
1585 #Non-coherent detection!
1591 powerNet = numpy.nansum(numpy.abs(voltsPShift[:,:,:])**2,0)
1586 powerNet = numpy.nansum(numpy.abs(voltsPShift[:,:,:])**2,0)
1592 #********** END OF COH/NON-COH POWER CALCULATION**********************
1587 #********** END OF COH/NON-COH POWER CALCULATION**********************
1593
1588
1594 #********** FIND THE NOISE LEVEL AND POSSIBLE METEORS ****************
1589 #********** FIND THE NOISE LEVEL AND POSSIBLE METEORS ****************
1595 #Get noise
1590 #Get noise
1596 noise, noise1 = self.__getNoise(powerNet, noise_timeStep, dataOut.timeInterval)
1591 noise, noise1 = self.__getNoise(powerNet, noise_timeStep, dataOut.timeInterval)
1597 # noise = self.getNoise1(powerNet, noise_timeStep, self.dataOut.timeInterval)
1592 # noise = self.getNoise1(powerNet, noise_timeStep, self.dataOut.timeInterval)
1598 #Get signal threshold
1593 #Get signal threshold
1599 signalThresh = noise_multiple*noise
1594 signalThresh = noise_multiple*noise
1600 #Meteor echoes detection
1595 #Meteor echoes detection
1601 listMeteors = self.__findMeteors(powerNet, signalThresh)
1596 listMeteors = self.__findMeteors(powerNet, signalThresh)
1602 #******* END OF NOISE LEVEL AND POSSIBLE METEORS CACULATION **********
1597 #******* END OF NOISE LEVEL AND POSSIBLE METEORS CACULATION **********
1603
1598
1604 #************** REMOVE MULTIPLE DETECTIONS (3.5) ***************************
1599 #************** REMOVE MULTIPLE DETECTIONS (3.5) ***************************
1605 #Parameters
1600 #Parameters
1606 heiRange = dataOut.getHeiRange()
1601 heiRange = dataOut.getHeiRange()
1607 rangeInterval = heiRange[1] - heiRange[0]
1602 rangeInterval = heiRange[1] - heiRange[0]
1608 rangeLimit = multDet_rangeLimit/rangeInterval
1603 rangeLimit = multDet_rangeLimit/rangeInterval
1609 timeLimit = multDet_timeLimit/dataOut.timeInterval
1604 timeLimit = multDet_timeLimit/dataOut.timeInterval
1610 #Multiple detection removals
1605 #Multiple detection removals
1611 listMeteors1 = self.__removeMultipleDetections(listMeteors, rangeLimit, timeLimit)
1606 listMeteors1 = self.__removeMultipleDetections(listMeteors, rangeLimit, timeLimit)
1612 #************ END OF REMOVE MULTIPLE DETECTIONS **********************
1607 #************ END OF REMOVE MULTIPLE DETECTIONS **********************
1613
1608
1614 #********************* METEOR REESTIMATION (3.7, 3.8, 3.9, 3.10) ********************
1609 #********************* METEOR REESTIMATION (3.7, 3.8, 3.9, 3.10) ********************
1615 #Parameters
1610 #Parameters
1616 phaseThresh = phaseThresh*numpy.pi/180
1611 phaseThresh = phaseThresh*numpy.pi/180
1617 thresh = [phaseThresh, noise_multiple, SNRThresh]
1612 thresh = [phaseThresh, noise_multiple, SNRThresh]
1618 #Meteor reestimation (Errors N 1, 6, 12, 17)
1613 #Meteor reestimation (Errors N 1, 6, 12, 17)
1619 listMeteors2, listMeteorsPower, listMeteorsVolts = self.__meteorReestimation(listMeteors1, voltsPShift, pairslist0, thresh, noise, dataOut.timeInterval, dataOut.frequency)
1614 listMeteors2, listMeteorsPower, listMeteorsVolts = self.__meteorReestimation(listMeteors1, voltsPShift, pairslist0, thresh, noise, dataOut.timeInterval, dataOut.frequency)
1620 # listMeteors2, listMeteorsPower, listMeteorsVolts = self.meteorReestimation3(listMeteors2, listMeteorsPower, listMeteorsVolts, voltsPShift, pairslist, thresh, noise)
1615 # listMeteors2, listMeteorsPower, listMeteorsVolts = self.meteorReestimation3(listMeteors2, listMeteorsPower, listMeteorsVolts, voltsPShift, pairslist, thresh, noise)
1621 #Estimation of decay times (Errors N 7, 8, 11)
1616 #Estimation of decay times (Errors N 7, 8, 11)
1622 listMeteors3 = self.__estimateDecayTime(listMeteors2, listMeteorsPower, dataOut.timeInterval, dataOut.frequency)
1617 listMeteors3 = self.__estimateDecayTime(listMeteors2, listMeteorsPower, dataOut.timeInterval, dataOut.frequency)
1623 #******************* END OF METEOR REESTIMATION *******************
1618 #******************* END OF METEOR REESTIMATION *******************
1624
1619
1625 #********************* METEOR PARAMETERS CALCULATION (3.11, 3.12, 3.13) **************************
1620 #********************* METEOR PARAMETERS CALCULATION (3.11, 3.12, 3.13) **************************
1626 #Calculating Radial Velocity (Error N 15)
1621 #Calculating Radial Velocity (Error N 15)
1627 radialStdThresh = 10
1622 radialStdThresh = 10
1628 listMeteors4 = self.__getRadialVelocity(listMeteors3, listMeteorsVolts, radialStdThresh, pairslist0, dataOut.timeInterval)
1623 listMeteors4 = self.__getRadialVelocity(listMeteors3, listMeteorsVolts, radialStdThresh, pairslist0, dataOut.timeInterval)
1629
1624
1630 if len(listMeteors4) > 0:
1625 if len(listMeteors4) > 0:
1631 #Setting New Array
1626 #Setting New Array
1632 date = dataOut.utctime
1627 date = dataOut.utctime
1633 arrayParameters = self.__setNewArrays(listMeteors4, date, heiRang)
1628 arrayParameters = self.__setNewArrays(listMeteors4, date, heiRang)
1634
1629
1635 #Correcting phase offset
1630 #Correcting phase offset
1636 if phaseOffsets != None:
1631 if phaseOffsets != None:
1637 phaseOffsets = numpy.array(phaseOffsets)*numpy.pi/180
1632 phaseOffsets = numpy.array(phaseOffsets)*numpy.pi/180
1638 arrayParameters[:,8:12] = numpy.unwrap(arrayParameters[:,8:12] + phaseOffsets)
1633 arrayParameters[:,8:12] = numpy.unwrap(arrayParameters[:,8:12] + phaseOffsets)
1639
1634
1640 #Second Pairslist
1635 #Second Pairslist
1641 pairsList = []
1636 pairsList = []
1642 pairx = (0,1)
1637 pairx = (0,1)
1643 pairy = (2,3)
1638 pairy = (2,3)
1644 pairsList.append(pairx)
1639 pairsList.append(pairx)
1645 pairsList.append(pairy)
1640 pairsList.append(pairy)
1646
1641
1647 jph = numpy.array([0,0,0,0])
1642 jph = numpy.array([0,0,0,0])
1648 h = (hmin,hmax)
1643 h = (hmin,hmax)
1649 arrayParameters = meteorOps.getMeteorParams(arrayParameters, azimuth, h, pairsList, distances, jph)
1644 arrayParameters = meteorOps.getMeteorParams(arrayParameters, azimuth, h, pairsList, distances, jph)
1650
1645
1651 # #Calculate AOA (Error N 3, 4)
1646 # #Calculate AOA (Error N 3, 4)
1652 # #JONES ET AL. 1998
1647 # #JONES ET AL. 1998
1653 # error = arrayParameters[:,-1]
1648 # error = arrayParameters[:,-1]
1654 # AOAthresh = numpy.pi/8
1649 # AOAthresh = numpy.pi/8
1655 # phases = -arrayParameters[:,9:13]
1650 # phases = -arrayParameters[:,9:13]
1656 # arrayParameters[:,4:7], arrayParameters[:,-1] = meteorOps.getAOA(phases, pairsList, error, AOAthresh, azimuth)
1651 # arrayParameters[:,4:7], arrayParameters[:,-1] = meteorOps.getAOA(phases, pairsList, error, AOAthresh, azimuth)
1657 #
1652 #
1658 # #Calculate Heights (Error N 13 and 14)
1653 # #Calculate Heights (Error N 13 and 14)
1659 # error = arrayParameters[:,-1]
1654 # error = arrayParameters[:,-1]
1660 # Ranges = arrayParameters[:,2]
1655 # Ranges = arrayParameters[:,2]
1661 # zenith = arrayParameters[:,5]
1656 # zenith = arrayParameters[:,5]
1662 # arrayParameters[:,3], arrayParameters[:,-1] = meteorOps.getHeights(Ranges, zenith, error, hmin, hmax)
1657 # arrayParameters[:,3], arrayParameters[:,-1] = meteorOps.getHeights(Ranges, zenith, error, hmin, hmax)
1663 # error = arrayParameters[:,-1]
1658 # error = arrayParameters[:,-1]
1664 #********************* END OF PARAMETERS CALCULATION **************************
1659 #********************* END OF PARAMETERS CALCULATION **************************
1665
1660
1666 #***************************+ PASS DATA TO NEXT STEP **********************
1661 #***************************+ PASS DATA TO NEXT STEP **********************
1667 # arrayFinal = arrayParameters.reshape((1,arrayParameters.shape[0],arrayParameters.shape[1]))
1662 # arrayFinal = arrayParameters.reshape((1,arrayParameters.shape[0],arrayParameters.shape[1]))
1668 dataOut.data_param = arrayParameters
1663 dataOut.data_param = arrayParameters
1669
1664
1670 if arrayParameters is None:
1665 if arrayParameters is None:
1671 dataOut.flagNoData = True
1666 dataOut.flagNoData = True
1672 else:
1667 else:
1673 dataOut.flagNoData = True
1668 dataOut.flagNoData = True
1674
1669
1675 return
1670 return
1676
1671
1677 def __getHardwarePhaseDiff(self, voltage0, pairslist, newheis, n):
1672 def __getHardwarePhaseDiff(self, voltage0, pairslist, newheis, n):
1678
1673
1679 minIndex = min(newheis[0])
1674 minIndex = min(newheis[0])
1680 maxIndex = max(newheis[0])
1675 maxIndex = max(newheis[0])
1681
1676
1682 voltage = voltage0[:,:,minIndex:maxIndex+1]
1677 voltage = voltage0[:,:,minIndex:maxIndex+1]
1683 nLength = voltage.shape[1]/n
1678 nLength = voltage.shape[1]/n
1684 nMin = 0
1679 nMin = 0
1685 nMax = 0
1680 nMax = 0
1686 phaseOffset = numpy.zeros((len(pairslist),n))
1681 phaseOffset = numpy.zeros((len(pairslist),n))
1687
1682
1688 for i in range(n):
1683 for i in range(n):
1689 nMax += nLength
1684 nMax += nLength
1690 phaseCCF = -numpy.angle(self.__calculateCCF(voltage[:,nMin:nMax,:], pairslist, [0]))
1685 phaseCCF = -numpy.angle(self.__calculateCCF(voltage[:,nMin:nMax,:], pairslist, [0]))
1691 phaseCCF = numpy.mean(phaseCCF, axis = 2)
1686 phaseCCF = numpy.mean(phaseCCF, axis = 2)
1692 phaseOffset[:,i] = phaseCCF.transpose()
1687 phaseOffset[:,i] = phaseCCF.transpose()
1693 nMin = nMax
1688 nMin = nMax
1694 # phaseDiff, phaseArrival = self.estimatePhaseDifference(voltage, pairslist)
1689 # phaseDiff, phaseArrival = self.estimatePhaseDifference(voltage, pairslist)
1695
1690
1696 #Remove Outliers
1691 #Remove Outliers
1697 factor = 2
1692 factor = 2
1698 wt = phaseOffset - signal.medfilt(phaseOffset,(1,5))
1693 wt = phaseOffset - signal.medfilt(phaseOffset,(1,5))
1699 dw = numpy.std(wt,axis = 1)
1694 dw = numpy.std(wt,axis = 1)
1700 dw = dw.reshape((dw.size,1))
1695 dw = dw.reshape((dw.size,1))
1701 ind = numpy.where(numpy.logical_or(wt>dw*factor,wt<-dw*factor))
1696 ind = numpy.where(numpy.logical_or(wt>dw*factor,wt<-dw*factor))
1702 phaseOffset[ind] = numpy.nan
1697 phaseOffset[ind] = numpy.nan
1703 phaseOffset = stats.nanmean(phaseOffset, axis=1)
1698 phaseOffset = stats.nanmean(phaseOffset, axis=1)
1704
1699
1705 return phaseOffset
1700 return phaseOffset
1706
1701
1707 def __shiftPhase(self, data, phaseShift):
1702 def __shiftPhase(self, data, phaseShift):
1708 #this will shift the phase of a complex number
1703 #this will shift the phase of a complex number
1709 dataShifted = numpy.abs(data) * numpy.exp((numpy.angle(data)+phaseShift)*1j)
1704 dataShifted = numpy.abs(data) * numpy.exp((numpy.angle(data)+phaseShift)*1j)
1710 return dataShifted
1705 return dataShifted
1711
1706
1712 def __estimatePhaseDifference(self, array, pairslist):
1707 def __estimatePhaseDifference(self, array, pairslist):
1713 nChannel = array.shape[0]
1708 nChannel = array.shape[0]
1714 nHeights = array.shape[2]
1709 nHeights = array.shape[2]
1715 numPairs = len(pairslist)
1710 numPairs = len(pairslist)
1716 # phaseCCF = numpy.zeros((nChannel, 5, nHeights))
1711 # phaseCCF = numpy.zeros((nChannel, 5, nHeights))
1717 phaseCCF = numpy.angle(self.__calculateCCF(array, pairslist, [-2,-1,0,1,2]))
1712 phaseCCF = numpy.angle(self.__calculateCCF(array, pairslist, [-2,-1,0,1,2]))
1718
1713
1719 #Correct phases
1714 #Correct phases
1720 derPhaseCCF = phaseCCF[:,1:,:] - phaseCCF[:,0:-1,:]
1715 derPhaseCCF = phaseCCF[:,1:,:] - phaseCCF[:,0:-1,:]
1721 indDer = numpy.where(numpy.abs(derPhaseCCF) > numpy.pi)
1716 indDer = numpy.where(numpy.abs(derPhaseCCF) > numpy.pi)
1722
1717
1723 if indDer[0].shape[0] > 0:
1718 if indDer[0].shape[0] > 0:
1724 for i in range(indDer[0].shape[0]):
1719 for i in range(indDer[0].shape[0]):
1725 signo = -numpy.sign(derPhaseCCF[indDer[0][i],indDer[1][i],indDer[2][i]])
1720 signo = -numpy.sign(derPhaseCCF[indDer[0][i],indDer[1][i],indDer[2][i]])
1726 phaseCCF[indDer[0][i],indDer[1][i]+1:,:] += signo*2*numpy.pi
1721 phaseCCF[indDer[0][i],indDer[1][i]+1:,:] += signo*2*numpy.pi
1727
1722
1728 # for j in range(numSides):
1723 # for j in range(numSides):
1729 # phaseCCFAux = self.calculateCCF(arrayCenter, arraySides[j,:,:], [-2,1,0,1,2])
1724 # phaseCCFAux = self.calculateCCF(arrayCenter, arraySides[j,:,:], [-2,1,0,1,2])
1730 # phaseCCF[j,:,:] = numpy.angle(phaseCCFAux)
1725 # phaseCCF[j,:,:] = numpy.angle(phaseCCFAux)
1731 #
1726 #
1732 #Linear
1727 #Linear
1733 phaseInt = numpy.zeros((numPairs,1))
1728 phaseInt = numpy.zeros((numPairs,1))
1734 angAllCCF = phaseCCF[:,[0,1,3,4],0]
1729 angAllCCF = phaseCCF[:,[0,1,3,4],0]
1735 for j in range(numPairs):
1730 for j in range(numPairs):
1736 fit = stats.linregress([-2,-1,1,2],angAllCCF[j,:])
1731 fit = stats.linregress([-2,-1,1,2],angAllCCF[j,:])
1737 phaseInt[j] = fit[1]
1732 phaseInt[j] = fit[1]
1738 #Phase Differences
1733 #Phase Differences
1739 phaseDiff = phaseInt - phaseCCF[:,2,:]
1734 phaseDiff = phaseInt - phaseCCF[:,2,:]
1740 phaseArrival = phaseInt.reshape(phaseInt.size)
1735 phaseArrival = phaseInt.reshape(phaseInt.size)
1741
1736
1742 #Dealias
1737 #Dealias
1743 phaseArrival = numpy.angle(numpy.exp(1j*phaseArrival))
1738 phaseArrival = numpy.angle(numpy.exp(1j*phaseArrival))
1744 # indAlias = numpy.where(phaseArrival > numpy.pi)
1739 # indAlias = numpy.where(phaseArrival > numpy.pi)
1745 # phaseArrival[indAlias] -= 2*numpy.pi
1740 # phaseArrival[indAlias] -= 2*numpy.pi
1746 # indAlias = numpy.where(phaseArrival < -numpy.pi)
1741 # indAlias = numpy.where(phaseArrival < -numpy.pi)
1747 # phaseArrival[indAlias] += 2*numpy.pi
1742 # phaseArrival[indAlias] += 2*numpy.pi
1748
1743
1749 return phaseDiff, phaseArrival
1744 return phaseDiff, phaseArrival
1750
1745
1751 def __coherentDetection(self, volts, timeSegment, timeInterval, pairslist, thresh):
1746 def __coherentDetection(self, volts, timeSegment, timeInterval, pairslist, thresh):
1752 #this function will run the coherent detection used in Holdworth et al. 2004 and return the net power
1747 #this function will run the coherent detection used in Holdworth et al. 2004 and return the net power
1753 #find the phase shifts of each channel over 1 second intervals
1748 #find the phase shifts of each channel over 1 second intervals
1754 #only look at ranges below the beacon signal
1749 #only look at ranges below the beacon signal
1755 numProfPerBlock = numpy.ceil(timeSegment/timeInterval)
1750 numProfPerBlock = numpy.ceil(timeSegment/timeInterval)
1756 numBlocks = int(volts.shape[1]/numProfPerBlock)
1751 numBlocks = int(volts.shape[1]/numProfPerBlock)
1757 numHeights = volts.shape[2]
1752 numHeights = volts.shape[2]
1758 nChannel = volts.shape[0]
1753 nChannel = volts.shape[0]
1759 voltsCohDet = volts.copy()
1754 voltsCohDet = volts.copy()
1760
1755
1761 pairsarray = numpy.array(pairslist)
1756 pairsarray = numpy.array(pairslist)
1762 indSides = pairsarray[:,1]
1757 indSides = pairsarray[:,1]
1763 # indSides = numpy.array(range(nChannel))
1758 # indSides = numpy.array(range(nChannel))
1764 # indSides = numpy.delete(indSides, indCenter)
1759 # indSides = numpy.delete(indSides, indCenter)
1765 #
1760 #
1766 # listCenter = numpy.array_split(volts[indCenter,:,:], numBlocks, 0)
1761 # listCenter = numpy.array_split(volts[indCenter,:,:], numBlocks, 0)
1767 listBlocks = numpy.array_split(volts, numBlocks, 1)
1762 listBlocks = numpy.array_split(volts, numBlocks, 1)
1768
1763
1769 startInd = 0
1764 startInd = 0
1770 endInd = 0
1765 endInd = 0
1771
1766
1772 for i in range(numBlocks):
1767 for i in range(numBlocks):
1773 startInd = endInd
1768 startInd = endInd
1774 endInd = endInd + listBlocks[i].shape[1]
1769 endInd = endInd + listBlocks[i].shape[1]
1775
1770
1776 arrayBlock = listBlocks[i]
1771 arrayBlock = listBlocks[i]
1777 # arrayBlockCenter = listCenter[i]
1772 # arrayBlockCenter = listCenter[i]
1778
1773
1779 #Estimate the Phase Difference
1774 #Estimate the Phase Difference
1780 phaseDiff, aux = self.__estimatePhaseDifference(arrayBlock, pairslist)
1775 phaseDiff, aux = self.__estimatePhaseDifference(arrayBlock, pairslist)
1781 #Phase Difference RMS
1776 #Phase Difference RMS
1782 arrayPhaseRMS = numpy.abs(phaseDiff)
1777 arrayPhaseRMS = numpy.abs(phaseDiff)
1783 phaseRMSaux = numpy.sum(arrayPhaseRMS < thresh,0)
1778 phaseRMSaux = numpy.sum(arrayPhaseRMS < thresh,0)
1784 indPhase = numpy.where(phaseRMSaux==4)
1779 indPhase = numpy.where(phaseRMSaux==4)
1785 #Shifting
1780 #Shifting
1786 if indPhase[0].shape[0] > 0:
1781 if indPhase[0].shape[0] > 0:
1787 for j in range(indSides.size):
1782 for j in range(indSides.size):
1788 arrayBlock[indSides[j],:,indPhase] = self.__shiftPhase(arrayBlock[indSides[j],:,indPhase], phaseDiff[j,indPhase].transpose())
1783 arrayBlock[indSides[j],:,indPhase] = self.__shiftPhase(arrayBlock[indSides[j],:,indPhase], phaseDiff[j,indPhase].transpose())
1789 voltsCohDet[:,startInd:endInd,:] = arrayBlock
1784 voltsCohDet[:,startInd:endInd,:] = arrayBlock
1790
1785
1791 return voltsCohDet
1786 return voltsCohDet
1792
1787
1793 def __calculateCCF(self, volts, pairslist ,laglist):
1788 def __calculateCCF(self, volts, pairslist ,laglist):
1794
1789
1795 nHeights = volts.shape[2]
1790 nHeights = volts.shape[2]
1796 nPoints = volts.shape[1]
1791 nPoints = volts.shape[1]
1797 voltsCCF = numpy.zeros((len(pairslist), len(laglist), nHeights),dtype = 'complex')
1792 voltsCCF = numpy.zeros((len(pairslist), len(laglist), nHeights),dtype = 'complex')
1798
1793
1799 for i in range(len(pairslist)):
1794 for i in range(len(pairslist)):
1800 volts1 = volts[pairslist[i][0]]
1795 volts1 = volts[pairslist[i][0]]
1801 volts2 = volts[pairslist[i][1]]
1796 volts2 = volts[pairslist[i][1]]
1802
1797
1803 for t in range(len(laglist)):
1798 for t in range(len(laglist)):
1804 idxT = laglist[t]
1799 idxT = laglist[t]
1805 if idxT >= 0:
1800 if idxT >= 0:
1806 vStacked = numpy.vstack((volts2[idxT:,:],
1801 vStacked = numpy.vstack((volts2[idxT:,:],
1807 numpy.zeros((idxT, nHeights),dtype='complex')))
1802 numpy.zeros((idxT, nHeights),dtype='complex')))
1808 else:
1803 else:
1809 vStacked = numpy.vstack((numpy.zeros((-idxT, nHeights),dtype='complex'),
1804 vStacked = numpy.vstack((numpy.zeros((-idxT, nHeights),dtype='complex'),
1810 volts2[:(nPoints + idxT),:]))
1805 volts2[:(nPoints + idxT),:]))
1811 voltsCCF[i,t,:] = numpy.sum((numpy.conjugate(volts1)*vStacked),axis=0)
1806 voltsCCF[i,t,:] = numpy.sum((numpy.conjugate(volts1)*vStacked),axis=0)
1812
1807
1813 vStacked = None
1808 vStacked = None
1814 return voltsCCF
1809 return voltsCCF
1815
1810
1816 def __getNoise(self, power, timeSegment, timeInterval):
1811 def __getNoise(self, power, timeSegment, timeInterval):
1817 numProfPerBlock = numpy.ceil(timeSegment/timeInterval)
1812 numProfPerBlock = numpy.ceil(timeSegment/timeInterval)
1818 numBlocks = int(power.shape[0]/numProfPerBlock)
1813 numBlocks = int(power.shape[0]/numProfPerBlock)
1819 numHeights = power.shape[1]
1814 numHeights = power.shape[1]
1820
1815
1821 listPower = numpy.array_split(power, numBlocks, 0)
1816 listPower = numpy.array_split(power, numBlocks, 0)
1822 noise = numpy.zeros((power.shape[0], power.shape[1]))
1817 noise = numpy.zeros((power.shape[0], power.shape[1]))
1823 noise1 = numpy.zeros((power.shape[0], power.shape[1]))
1818 noise1 = numpy.zeros((power.shape[0], power.shape[1]))
1824
1819
1825 startInd = 0
1820 startInd = 0
1826 endInd = 0
1821 endInd = 0
1827
1822
1828 for i in range(numBlocks): #split por canal
1823 for i in range(numBlocks): #split por canal
1829 startInd = endInd
1824 startInd = endInd
1830 endInd = endInd + listPower[i].shape[0]
1825 endInd = endInd + listPower[i].shape[0]
1831
1826
1832 arrayBlock = listPower[i]
1827 arrayBlock = listPower[i]
1833 noiseAux = numpy.mean(arrayBlock, 0)
1828 noiseAux = numpy.mean(arrayBlock, 0)
1834 # noiseAux = numpy.median(noiseAux)
1829 # noiseAux = numpy.median(noiseAux)
1835 # noiseAux = numpy.mean(arrayBlock)
1830 # noiseAux = numpy.mean(arrayBlock)
1836 noise[startInd:endInd,:] = noise[startInd:endInd,:] + noiseAux
1831 noise[startInd:endInd,:] = noise[startInd:endInd,:] + noiseAux
1837
1832
1838 noiseAux1 = numpy.mean(arrayBlock)
1833 noiseAux1 = numpy.mean(arrayBlock)
1839 noise1[startInd:endInd,:] = noise1[startInd:endInd,:] + noiseAux1
1834 noise1[startInd:endInd,:] = noise1[startInd:endInd,:] + noiseAux1
1840
1835
1841 return noise, noise1
1836 return noise, noise1
1842
1837
1843 def __findMeteors(self, power, thresh):
1838 def __findMeteors(self, power, thresh):
1844 nProf = power.shape[0]
1839 nProf = power.shape[0]
1845 nHeights = power.shape[1]
1840 nHeights = power.shape[1]
1846 listMeteors = []
1841 listMeteors = []
1847
1842
1848 for i in range(nHeights):
1843 for i in range(nHeights):
1849 powerAux = power[:,i]
1844 powerAux = power[:,i]
1850 threshAux = thresh[:,i]
1845 threshAux = thresh[:,i]
1851
1846
1852 indUPthresh = numpy.where(powerAux > threshAux)[0]
1847 indUPthresh = numpy.where(powerAux > threshAux)[0]
1853 indDNthresh = numpy.where(powerAux <= threshAux)[0]
1848 indDNthresh = numpy.where(powerAux <= threshAux)[0]
1854
1849
1855 j = 0
1850 j = 0
1856
1851
1857 while (j < indUPthresh.size - 2):
1852 while (j < indUPthresh.size - 2):
1858 if (indUPthresh[j + 2] == indUPthresh[j] + 2):
1853 if (indUPthresh[j + 2] == indUPthresh[j] + 2):
1859 indDNAux = numpy.where(indDNthresh > indUPthresh[j])
1854 indDNAux = numpy.where(indDNthresh > indUPthresh[j])
1860 indDNthresh = indDNthresh[indDNAux]
1855 indDNthresh = indDNthresh[indDNAux]
1861
1856
1862 if (indDNthresh.size > 0):
1857 if (indDNthresh.size > 0):
1863 indEnd = indDNthresh[0] - 1
1858 indEnd = indDNthresh[0] - 1
1864 indInit = indUPthresh[j] if isinstance(indUPthresh[j], (int, float)) else indUPthresh[j][0] ##CHECK!!!!
1859 indInit = indUPthresh[j] if isinstance(indUPthresh[j], (int, float)) else indUPthresh[j][0] ##CHECK!!!!
1865
1860
1866 meteor = powerAux[indInit:indEnd + 1]
1861 meteor = powerAux[indInit:indEnd + 1]
1867 indPeak = meteor.argmax() + indInit
1862 indPeak = meteor.argmax() + indInit
1868 FLA = sum(numpy.conj(meteor)*numpy.hstack((meteor[1:],0)))
1863 FLA = sum(numpy.conj(meteor)*numpy.hstack((meteor[1:],0)))
1869
1864
1870 listMeteors.append(numpy.array([i,indInit,indPeak,indEnd,FLA])) #CHEQUEAR!!!!!
1865 listMeteors.append(numpy.array([i,indInit,indPeak,indEnd,FLA])) #CHEQUEAR!!!!!
1871 j = numpy.where(indUPthresh == indEnd)[0] + 1
1866 j = numpy.where(indUPthresh == indEnd)[0] + 1
1872 else: j+=1
1867 else: j+=1
1873 else: j+=1
1868 else: j+=1
1874
1869
1875 return listMeteors
1870 return listMeteors
1876
1871
1877 def __removeMultipleDetections(self,listMeteors, rangeLimit, timeLimit):
1872 def __removeMultipleDetections(self,listMeteors, rangeLimit, timeLimit):
1878
1873
1879 arrayMeteors = numpy.asarray(listMeteors)
1874 arrayMeteors = numpy.asarray(listMeteors)
1880 listMeteors1 = []
1875 listMeteors1 = []
1881
1876
1882 while arrayMeteors.shape[0] > 0:
1877 while arrayMeteors.shape[0] > 0:
1883 FLAs = arrayMeteors[:,4]
1878 FLAs = arrayMeteors[:,4]
1884 maxFLA = FLAs.argmax()
1879 maxFLA = FLAs.argmax()
1885 listMeteors1.append(arrayMeteors[maxFLA,:])
1880 listMeteors1.append(arrayMeteors[maxFLA,:])
1886
1881
1887 MeteorInitTime = arrayMeteors[maxFLA,1]
1882 MeteorInitTime = arrayMeteors[maxFLA,1]
1888 MeteorEndTime = arrayMeteors[maxFLA,3]
1883 MeteorEndTime = arrayMeteors[maxFLA,3]
1889 MeteorHeight = arrayMeteors[maxFLA,0]
1884 MeteorHeight = arrayMeteors[maxFLA,0]
1890
1885
1891 #Check neighborhood
1886 #Check neighborhood
1892 maxHeightIndex = MeteorHeight + rangeLimit
1887 maxHeightIndex = MeteorHeight + rangeLimit
1893 minHeightIndex = MeteorHeight - rangeLimit
1888 minHeightIndex = MeteorHeight - rangeLimit
1894 minTimeIndex = MeteorInitTime - timeLimit
1889 minTimeIndex = MeteorInitTime - timeLimit
1895 maxTimeIndex = MeteorEndTime + timeLimit
1890 maxTimeIndex = MeteorEndTime + timeLimit
1896
1891
1897 #Check Heights
1892 #Check Heights
1898 indHeight = numpy.logical_and(arrayMeteors[:,0] >= minHeightIndex, arrayMeteors[:,0] <= maxHeightIndex)
1893 indHeight = numpy.logical_and(arrayMeteors[:,0] >= minHeightIndex, arrayMeteors[:,0] <= maxHeightIndex)
1899 indTime = numpy.logical_and(arrayMeteors[:,3] >= minTimeIndex, arrayMeteors[:,1] <= maxTimeIndex)
1894 indTime = numpy.logical_and(arrayMeteors[:,3] >= minTimeIndex, arrayMeteors[:,1] <= maxTimeIndex)
1900 indBoth = numpy.where(numpy.logical_and(indTime,indHeight))
1895 indBoth = numpy.where(numpy.logical_and(indTime,indHeight))
1901
1896
1902 arrayMeteors = numpy.delete(arrayMeteors, indBoth, axis = 0)
1897 arrayMeteors = numpy.delete(arrayMeteors, indBoth, axis = 0)
1903
1898
1904 return listMeteors1
1899 return listMeteors1
1905
1900
1906 def __meteorReestimation(self, listMeteors, volts, pairslist, thresh, noise, timeInterval,frequency):
1901 def __meteorReestimation(self, listMeteors, volts, pairslist, thresh, noise, timeInterval,frequency):
1907 numHeights = volts.shape[2]
1902 numHeights = volts.shape[2]
1908 nChannel = volts.shape[0]
1903 nChannel = volts.shape[0]
1909
1904
1910 thresholdPhase = thresh[0]
1905 thresholdPhase = thresh[0]
1911 thresholdNoise = thresh[1]
1906 thresholdNoise = thresh[1]
1912 thresholdDB = float(thresh[2])
1907 thresholdDB = float(thresh[2])
1913
1908
1914 thresholdDB1 = 10**(thresholdDB/10)
1909 thresholdDB1 = 10**(thresholdDB/10)
1915 pairsarray = numpy.array(pairslist)
1910 pairsarray = numpy.array(pairslist)
1916 indSides = pairsarray[:,1]
1911 indSides = pairsarray[:,1]
1917
1912
1918 pairslist1 = list(pairslist)
1913 pairslist1 = list(pairslist)
1919 pairslist1.append((0,4))
1914 pairslist1.append((0,4))
1920 pairslist1.append((1,3))
1915 pairslist1.append((1,3))
1921
1916
1922 listMeteors1 = []
1917 listMeteors1 = []
1923 listPowerSeries = []
1918 listPowerSeries = []
1924 listVoltageSeries = []
1919 listVoltageSeries = []
1925 #volts has the war data
1920 #volts has the war data
1926
1921
1927 if frequency == 30.175e6:
1922 if frequency == 30.175e6:
1928 timeLag = 45*10**-3
1923 timeLag = 45*10**-3
1929 else:
1924 else:
1930 timeLag = 15*10**-3
1925 timeLag = 15*10**-3
1931 lag = int(numpy.ceil(timeLag/timeInterval))
1926 lag = int(numpy.ceil(timeLag/timeInterval))
1932
1927
1933 for i in range(len(listMeteors)):
1928 for i in range(len(listMeteors)):
1934
1929
1935 ###################### 3.6 - 3.7 PARAMETERS REESTIMATION #########################
1930 ###################### 3.6 - 3.7 PARAMETERS REESTIMATION #########################
1936 meteorAux = numpy.zeros(16)
1931 meteorAux = numpy.zeros(16)
1937
1932
1938 #Loading meteor Data (mHeight, mStart, mPeak, mEnd)
1933 #Loading meteor Data (mHeight, mStart, mPeak, mEnd)
1939 mHeight = int(listMeteors[i][0])
1934 mHeight = int(listMeteors[i][0])
1940 mStart = int(listMeteors[i][1])
1935 mStart = int(listMeteors[i][1])
1941 mPeak = int(listMeteors[i][2])
1936 mPeak = int(listMeteors[i][2])
1942 mEnd = int(listMeteors[i][3])
1937 mEnd = int(listMeteors[i][3])
1943
1938
1944 #get the volt data between the start and end times of the meteor
1939 #get the volt data between the start and end times of the meteor
1945 meteorVolts = volts[:,mStart:mEnd+1,mHeight]
1940 meteorVolts = volts[:,mStart:mEnd+1,mHeight]
1946 meteorVolts = meteorVolts.reshape(meteorVolts.shape[0], meteorVolts.shape[1], 1)
1941 meteorVolts = meteorVolts.reshape(meteorVolts.shape[0], meteorVolts.shape[1], 1)
1947
1942
1948 #3.6. Phase Difference estimation
1943 #3.6. Phase Difference estimation
1949 phaseDiff, aux = self.__estimatePhaseDifference(meteorVolts, pairslist)
1944 phaseDiff, aux = self.__estimatePhaseDifference(meteorVolts, pairslist)
1950
1945
1951 #3.7. Phase difference removal & meteor start, peak and end times reestimated
1946 #3.7. Phase difference removal & meteor start, peak and end times reestimated
1952 #meteorVolts0.- all Channels, all Profiles
1947 #meteorVolts0.- all Channels, all Profiles
1953 meteorVolts0 = volts[:,:,mHeight]
1948 meteorVolts0 = volts[:,:,mHeight]
1954 meteorThresh = noise[:,mHeight]*thresholdNoise
1949 meteorThresh = noise[:,mHeight]*thresholdNoise
1955 meteorNoise = noise[:,mHeight]
1950 meteorNoise = noise[:,mHeight]
1956 meteorVolts0[indSides,:] = self.__shiftPhase(meteorVolts0[indSides,:], phaseDiff) #Phase Shifting
1951 meteorVolts0[indSides,:] = self.__shiftPhase(meteorVolts0[indSides,:], phaseDiff) #Phase Shifting
1957 powerNet0 = numpy.nansum(numpy.abs(meteorVolts0)**2, axis = 0) #Power
1952 powerNet0 = numpy.nansum(numpy.abs(meteorVolts0)**2, axis = 0) #Power
1958
1953
1959 #Times reestimation
1954 #Times reestimation
1960 mStart1 = numpy.where(powerNet0[:mPeak] < meteorThresh[:mPeak])[0]
1955 mStart1 = numpy.where(powerNet0[:mPeak] < meteorThresh[:mPeak])[0]
1961 if mStart1.size > 0:
1956 if mStart1.size > 0:
1962 mStart1 = mStart1[-1] + 1
1957 mStart1 = mStart1[-1] + 1
1963
1958
1964 else:
1959 else:
1965 mStart1 = mPeak
1960 mStart1 = mPeak
1966
1961
1967 mEnd1 = numpy.where(powerNet0[mPeak:] < meteorThresh[mPeak:])[0][0] + mPeak - 1
1962 mEnd1 = numpy.where(powerNet0[mPeak:] < meteorThresh[mPeak:])[0][0] + mPeak - 1
1968 mEndDecayTime1 = numpy.where(powerNet0[mPeak:] < meteorNoise[mPeak:])[0]
1963 mEndDecayTime1 = numpy.where(powerNet0[mPeak:] < meteorNoise[mPeak:])[0]
1969 if mEndDecayTime1.size == 0:
1964 if mEndDecayTime1.size == 0:
1970 mEndDecayTime1 = powerNet0.size
1965 mEndDecayTime1 = powerNet0.size
1971 else:
1966 else:
1972 mEndDecayTime1 = mEndDecayTime1[0] + mPeak - 1
1967 mEndDecayTime1 = mEndDecayTime1[0] + mPeak - 1
1973 # mPeak1 = meteorVolts0[mStart1:mEnd1 + 1].argmax()
1968 # mPeak1 = meteorVolts0[mStart1:mEnd1 + 1].argmax()
1974
1969
1975 #meteorVolts1.- all Channels, from start to end
1970 #meteorVolts1.- all Channels, from start to end
1976 meteorVolts1 = meteorVolts0[:,mStart1:mEnd1 + 1]
1971 meteorVolts1 = meteorVolts0[:,mStart1:mEnd1 + 1]
1977 meteorVolts2 = meteorVolts0[:,mPeak + lag:mEnd1 + 1]
1972 meteorVolts2 = meteorVolts0[:,mPeak + lag:mEnd1 + 1]
1978 if meteorVolts2.shape[1] == 0:
1973 if meteorVolts2.shape[1] == 0:
1979 meteorVolts2 = meteorVolts0[:,mPeak:mEnd1 + 1]
1974 meteorVolts2 = meteorVolts0[:,mPeak:mEnd1 + 1]
1980 meteorVolts1 = meteorVolts1.reshape(meteorVolts1.shape[0], meteorVolts1.shape[1], 1)
1975 meteorVolts1 = meteorVolts1.reshape(meteorVolts1.shape[0], meteorVolts1.shape[1], 1)
1981 meteorVolts2 = meteorVolts2.reshape(meteorVolts2.shape[0], meteorVolts2.shape[1], 1)
1976 meteorVolts2 = meteorVolts2.reshape(meteorVolts2.shape[0], meteorVolts2.shape[1], 1)
1982 ##################### END PARAMETERS REESTIMATION #########################
1977 ##################### END PARAMETERS REESTIMATION #########################
1983
1978
1984 ##################### 3.8 PHASE DIFFERENCE REESTIMATION ########################
1979 ##################### 3.8 PHASE DIFFERENCE REESTIMATION ########################
1985 # if mEnd1 - mStart1 > 4: #Error Number 6: echo less than 5 samples long; too short for analysis
1980 # if mEnd1 - mStart1 > 4: #Error Number 6: echo less than 5 samples long; too short for analysis
1986 if meteorVolts2.shape[1] > 0:
1981 if meteorVolts2.shape[1] > 0:
1987 #Phase Difference re-estimation
1982 #Phase Difference re-estimation
1988 phaseDiff1, phaseDiffint = self.__estimatePhaseDifference(meteorVolts2, pairslist1) #Phase Difference Estimation
1983 phaseDiff1, phaseDiffint = self.__estimatePhaseDifference(meteorVolts2, pairslist1) #Phase Difference Estimation
1989 # phaseDiff1, phaseDiffint = self.estimatePhaseDifference(meteorVolts2, pairslist)
1984 # phaseDiff1, phaseDiffint = self.estimatePhaseDifference(meteorVolts2, pairslist)
1990 meteorVolts2 = meteorVolts2.reshape(meteorVolts2.shape[0], meteorVolts2.shape[1])
1985 meteorVolts2 = meteorVolts2.reshape(meteorVolts2.shape[0], meteorVolts2.shape[1])
1991 phaseDiff11 = numpy.reshape(phaseDiff1, (phaseDiff1.shape[0],1))
1986 phaseDiff11 = numpy.reshape(phaseDiff1, (phaseDiff1.shape[0],1))
1992 meteorVolts2[indSides,:] = self.__shiftPhase(meteorVolts2[indSides,:], phaseDiff11[0:4]) #Phase Shifting
1987 meteorVolts2[indSides,:] = self.__shiftPhase(meteorVolts2[indSides,:], phaseDiff11[0:4]) #Phase Shifting
1993
1988
1994 #Phase Difference RMS
1989 #Phase Difference RMS
1995 phaseRMS1 = numpy.sqrt(numpy.mean(numpy.square(phaseDiff1)))
1990 phaseRMS1 = numpy.sqrt(numpy.mean(numpy.square(phaseDiff1)))
1996 powerNet1 = numpy.nansum(numpy.abs(meteorVolts1[:,:])**2,0)
1991 powerNet1 = numpy.nansum(numpy.abs(meteorVolts1[:,:])**2,0)
1997 #Data from Meteor
1992 #Data from Meteor
1998 mPeak1 = powerNet1.argmax() + mStart1
1993 mPeak1 = powerNet1.argmax() + mStart1
1999 mPeakPower1 = powerNet1.max()
1994 mPeakPower1 = powerNet1.max()
2000 noiseAux = sum(noise[mStart1:mEnd1 + 1,mHeight])
1995 noiseAux = sum(noise[mStart1:mEnd1 + 1,mHeight])
2001 mSNR1 = (sum(powerNet1)-noiseAux)/noiseAux
1996 mSNR1 = (sum(powerNet1)-noiseAux)/noiseAux
2002 Meteor1 = numpy.array([mHeight, mStart1, mPeak1, mEnd1, mPeakPower1, mSNR1, phaseRMS1])
1997 Meteor1 = numpy.array([mHeight, mStart1, mPeak1, mEnd1, mPeakPower1, mSNR1, phaseRMS1])
2003 Meteor1 = numpy.hstack((Meteor1,phaseDiffint))
1998 Meteor1 = numpy.hstack((Meteor1,phaseDiffint))
2004 PowerSeries = powerNet0[mStart1:mEndDecayTime1 + 1]
1999 PowerSeries = powerNet0[mStart1:mEndDecayTime1 + 1]
2005 #Vectorize
2000 #Vectorize
2006 meteorAux[0:7] = [mHeight, mStart1, mPeak1, mEnd1, mPeakPower1, mSNR1, phaseRMS1]
2001 meteorAux[0:7] = [mHeight, mStart1, mPeak1, mEnd1, mPeakPower1, mSNR1, phaseRMS1]
2007 meteorAux[7:11] = phaseDiffint[0:4]
2002 meteorAux[7:11] = phaseDiffint[0:4]
2008
2003
2009 #Rejection Criterions
2004 #Rejection Criterions
2010 if phaseRMS1 > thresholdPhase: #Error Number 17: Phase variation
2005 if phaseRMS1 > thresholdPhase: #Error Number 17: Phase variation
2011 meteorAux[-1] = 17
2006 meteorAux[-1] = 17
2012 elif mSNR1 < thresholdDB1: #Error Number 1: SNR < threshold dB
2007 elif mSNR1 < thresholdDB1: #Error Number 1: SNR < threshold dB
2013 meteorAux[-1] = 1
2008 meteorAux[-1] = 1
2014
2009
2015
2010
2016 else:
2011 else:
2017 meteorAux[0:4] = [mHeight, mStart, mPeak, mEnd]
2012 meteorAux[0:4] = [mHeight, mStart, mPeak, mEnd]
2018 meteorAux[-1] = 6 #Error Number 6: echo less than 5 samples long; too short for analysis
2013 meteorAux[-1] = 6 #Error Number 6: echo less than 5 samples long; too short for analysis
2019 PowerSeries = 0
2014 PowerSeries = 0
2020
2015
2021 listMeteors1.append(meteorAux)
2016 listMeteors1.append(meteorAux)
2022 listPowerSeries.append(PowerSeries)
2017 listPowerSeries.append(PowerSeries)
2023 listVoltageSeries.append(meteorVolts1)
2018 listVoltageSeries.append(meteorVolts1)
2024
2019
2025 return listMeteors1, listPowerSeries, listVoltageSeries
2020 return listMeteors1, listPowerSeries, listVoltageSeries
2026
2021
2027 def __estimateDecayTime(self, listMeteors, listPower, timeInterval, frequency):
2022 def __estimateDecayTime(self, listMeteors, listPower, timeInterval, frequency):
2028
2023
2029 threshError = 10
2024 threshError = 10
2030 #Depending if it is 30 or 50 MHz
2025 #Depending if it is 30 or 50 MHz
2031 if frequency == 30.175e6:
2026 if frequency == 30.175e6:
2032 timeLag = 45*10**-3
2027 timeLag = 45*10**-3
2033 else:
2028 else:
2034 timeLag = 15*10**-3
2029 timeLag = 15*10**-3
2035 lag = int(numpy.ceil(timeLag/timeInterval))
2030 lag = int(numpy.ceil(timeLag/timeInterval))
2036
2031
2037 listMeteors1 = []
2032 listMeteors1 = []
2038
2033
2039 for i in range(len(listMeteors)):
2034 for i in range(len(listMeteors)):
2040 meteorPower = listPower[i]
2035 meteorPower = listPower[i]
2041 meteorAux = listMeteors[i]
2036 meteorAux = listMeteors[i]
2042
2037
2043 if meteorAux[-1] == 0:
2038 if meteorAux[-1] == 0:
2044
2039
2045 try:
2040 try:
2046 indmax = meteorPower.argmax()
2041 indmax = meteorPower.argmax()
2047 indlag = indmax + lag
2042 indlag = indmax + lag
2048
2043
2049 y = meteorPower[indlag:]
2044 y = meteorPower[indlag:]
2050 x = numpy.arange(0, y.size)*timeLag
2045 x = numpy.arange(0, y.size)*timeLag
2051
2046
2052 #first guess
2047 #first guess
2053 a = y[0]
2048 a = y[0]
2054 tau = timeLag
2049 tau = timeLag
2055 #exponential fit
2050 #exponential fit
2056 popt, pcov = optimize.curve_fit(self.__exponential_function, x, y, p0 = [a, tau])
2051 popt, pcov = optimize.curve_fit(self.__exponential_function, x, y, p0 = [a, tau])
2057 y1 = self.__exponential_function(x, *popt)
2052 y1 = self.__exponential_function(x, *popt)
2058 #error estimation
2053 #error estimation
2059 error = sum((y - y1)**2)/(numpy.var(y)*(y.size - popt.size))
2054 error = sum((y - y1)**2)/(numpy.var(y)*(y.size - popt.size))
2060
2055
2061 decayTime = popt[1]
2056 decayTime = popt[1]
2062 riseTime = indmax*timeInterval
2057 riseTime = indmax*timeInterval
2063 meteorAux[11:13] = [decayTime, error]
2058 meteorAux[11:13] = [decayTime, error]
2064
2059
2065 #Table items 7, 8 and 11
2060 #Table items 7, 8 and 11
2066 if (riseTime > 0.3): #Number 7: Echo rise exceeds 0.3s
2061 if (riseTime > 0.3): #Number 7: Echo rise exceeds 0.3s
2067 meteorAux[-1] = 7
2062 meteorAux[-1] = 7
2068 elif (decayTime < 2*riseTime) : #Number 8: Echo decay time less than than twice rise time
2063 elif (decayTime < 2*riseTime) : #Number 8: Echo decay time less than than twice rise time
2069 meteorAux[-1] = 8
2064 meteorAux[-1] = 8
2070 if (error > threshError): #Number 11: Poor fit to amplitude for estimation of decay time
2065 if (error > threshError): #Number 11: Poor fit to amplitude for estimation of decay time
2071 meteorAux[-1] = 11
2066 meteorAux[-1] = 11
2072
2067
2073
2068
2074 except:
2069 except:
2075 meteorAux[-1] = 11
2070 meteorAux[-1] = 11
2076
2071
2077
2072
2078 listMeteors1.append(meteorAux)
2073 listMeteors1.append(meteorAux)
2079
2074
2080 return listMeteors1
2075 return listMeteors1
2081
2076
2082 #Exponential Function
2077 #Exponential Function
2083
2078
2084 def __exponential_function(self, x, a, tau):
2079 def __exponential_function(self, x, a, tau):
2085 y = a*numpy.exp(-x/tau)
2080 y = a*numpy.exp(-x/tau)
2086 return y
2081 return y
2087
2082
2088 def __getRadialVelocity(self, listMeteors, listVolts, radialStdThresh, pairslist, timeInterval):
2083 def __getRadialVelocity(self, listMeteors, listVolts, radialStdThresh, pairslist, timeInterval):
2089
2084
2090 pairslist1 = list(pairslist)
2085 pairslist1 = list(pairslist)
2091 pairslist1.append((0,4))
2086 pairslist1.append((0,4))
2092 pairslist1.append((1,3))
2087 pairslist1.append((1,3))
2093 numPairs = len(pairslist1)
2088 numPairs = len(pairslist1)
2094 #Time Lag
2089 #Time Lag
2095 timeLag = 45*10**-3
2090 timeLag = 45*10**-3
2096 c = 3e8
2091 c = 3e8
2097 lag = numpy.ceil(timeLag/timeInterval)
2092 lag = numpy.ceil(timeLag/timeInterval)
2098 freq = 30.175e6
2093 freq = 30.175e6
2099
2094
2100 listMeteors1 = []
2095 listMeteors1 = []
2101
2096
2102 for i in range(len(listMeteors)):
2097 for i in range(len(listMeteors)):
2103 meteorAux = listMeteors[i]
2098 meteorAux = listMeteors[i]
2104 if meteorAux[-1] == 0:
2099 if meteorAux[-1] == 0:
2105 mStart = listMeteors[i][1]
2100 mStart = listMeteors[i][1]
2106 mPeak = listMeteors[i][2]
2101 mPeak = listMeteors[i][2]
2107 mLag = mPeak - mStart + lag
2102 mLag = mPeak - mStart + lag
2108
2103
2109 #get the volt data between the start and end times of the meteor
2104 #get the volt data between the start and end times of the meteor
2110 meteorVolts = listVolts[i]
2105 meteorVolts = listVolts[i]
2111 meteorVolts = meteorVolts.reshape(meteorVolts.shape[0], meteorVolts.shape[1], 1)
2106 meteorVolts = meteorVolts.reshape(meteorVolts.shape[0], meteorVolts.shape[1], 1)
2112
2107
2113 #Get CCF
2108 #Get CCF
2114 allCCFs = self.__calculateCCF(meteorVolts, pairslist1, [-2,-1,0,1,2])
2109 allCCFs = self.__calculateCCF(meteorVolts, pairslist1, [-2,-1,0,1,2])
2115
2110
2116 #Method 2
2111 #Method 2
2117 slopes = numpy.zeros(numPairs)
2112 slopes = numpy.zeros(numPairs)
2118 time = numpy.array([-2,-1,1,2])*timeInterval
2113 time = numpy.array([-2,-1,1,2])*timeInterval
2119 angAllCCF = numpy.angle(allCCFs[:,[0,4,2,3],0])
2114 angAllCCF = numpy.angle(allCCFs[:,[0,4,2,3],0])
2120
2115
2121 #Correct phases
2116 #Correct phases
2122 derPhaseCCF = angAllCCF[:,1:] - angAllCCF[:,0:-1]
2117 derPhaseCCF = angAllCCF[:,1:] - angAllCCF[:,0:-1]
2123 indDer = numpy.where(numpy.abs(derPhaseCCF) > numpy.pi)
2118 indDer = numpy.where(numpy.abs(derPhaseCCF) > numpy.pi)
2124
2119
2125 if indDer[0].shape[0] > 0:
2120 if indDer[0].shape[0] > 0:
2126 for i in range(indDer[0].shape[0]):
2121 for i in range(indDer[0].shape[0]):
2127 signo = -numpy.sign(derPhaseCCF[indDer[0][i],indDer[1][i]])
2122 signo = -numpy.sign(derPhaseCCF[indDer[0][i],indDer[1][i]])
2128 angAllCCF[indDer[0][i],indDer[1][i]+1:] += signo*2*numpy.pi
2123 angAllCCF[indDer[0][i],indDer[1][i]+1:] += signo*2*numpy.pi
2129
2124
2130 # fit = scipy.stats.linregress(numpy.array([-2,-1,1,2])*timeInterval, numpy.array([phaseLagN2s[i],phaseLagN1s[i],phaseLag1s[i],phaseLag2s[i]]))
2125 # fit = scipy.stats.linregress(numpy.array([-2,-1,1,2])*timeInterval, numpy.array([phaseLagN2s[i],phaseLagN1s[i],phaseLag1s[i],phaseLag2s[i]]))
2131 for j in range(numPairs):
2126 for j in range(numPairs):
2132 fit = stats.linregress(time, angAllCCF[j,:])
2127 fit = stats.linregress(time, angAllCCF[j,:])
2133 slopes[j] = fit[0]
2128 slopes[j] = fit[0]
2134
2129
2135 #Remove Outlier
2130 #Remove Outlier
2136 # indOut = numpy.argmax(numpy.abs(slopes - numpy.mean(slopes)))
2131 # indOut = numpy.argmax(numpy.abs(slopes - numpy.mean(slopes)))
2137 # slopes = numpy.delete(slopes,indOut)
2132 # slopes = numpy.delete(slopes,indOut)
2138 # indOut = numpy.argmax(numpy.abs(slopes - numpy.mean(slopes)))
2133 # indOut = numpy.argmax(numpy.abs(slopes - numpy.mean(slopes)))
2139 # slopes = numpy.delete(slopes,indOut)
2134 # slopes = numpy.delete(slopes,indOut)
2140
2135
2141 radialVelocity = -numpy.mean(slopes)*(0.25/numpy.pi)*(c/freq)
2136 radialVelocity = -numpy.mean(slopes)*(0.25/numpy.pi)*(c/freq)
2142 radialError = numpy.std(slopes)*(0.25/numpy.pi)*(c/freq)
2137 radialError = numpy.std(slopes)*(0.25/numpy.pi)*(c/freq)
2143 meteorAux[-2] = radialError
2138 meteorAux[-2] = radialError
2144 meteorAux[-3] = radialVelocity
2139 meteorAux[-3] = radialVelocity
2145
2140
2146 #Setting Error
2141 #Setting Error
2147 #Number 15: Radial Drift velocity or projected horizontal velocity exceeds 200 m/s
2142 #Number 15: Radial Drift velocity or projected horizontal velocity exceeds 200 m/s
2148 if numpy.abs(radialVelocity) > 200:
2143 if numpy.abs(radialVelocity) > 200:
2149 meteorAux[-1] = 15
2144 meteorAux[-1] = 15
2150 #Number 12: Poor fit to CCF variation for estimation of radial drift velocity
2145 #Number 12: Poor fit to CCF variation for estimation of radial drift velocity
2151 elif radialError > radialStdThresh:
2146 elif radialError > radialStdThresh:
2152 meteorAux[-1] = 12
2147 meteorAux[-1] = 12
2153
2148
2154 listMeteors1.append(meteorAux)
2149 listMeteors1.append(meteorAux)
2155 return listMeteors1
2150 return listMeteors1
2156
2151
2157 def __setNewArrays(self, listMeteors, date, heiRang):
2152 def __setNewArrays(self, listMeteors, date, heiRang):
2158
2153
2159 #New arrays
2154 #New arrays
2160 arrayMeteors = numpy.array(listMeteors)
2155 arrayMeteors = numpy.array(listMeteors)
2161 arrayParameters = numpy.zeros((len(listMeteors), 13))
2156 arrayParameters = numpy.zeros((len(listMeteors), 13))
2162
2157
2163 #Date inclusion
2158 #Date inclusion
2164 # date = re.findall(r'\((.*?)\)', date)
2159 # date = re.findall(r'\((.*?)\)', date)
2165 # date = date[0].split(',')
2160 # date = date[0].split(',')
2166 # date = map(int, date)
2161 # date = map(int, date)
2167 #
2162 #
2168 # if len(date)<6:
2163 # if len(date)<6:
2169 # date.append(0)
2164 # date.append(0)
2170 #
2165 #
2171 # date = [date[0]*10000 + date[1]*100 + date[2], date[3]*10000 + date[4]*100 + date[5]]
2166 # date = [date[0]*10000 + date[1]*100 + date[2], date[3]*10000 + date[4]*100 + date[5]]
2172 # arrayDate = numpy.tile(date, (len(listMeteors), 1))
2167 # arrayDate = numpy.tile(date, (len(listMeteors), 1))
2173 arrayDate = numpy.tile(date, (len(listMeteors)))
2168 arrayDate = numpy.tile(date, (len(listMeteors)))
2174
2169
2175 #Meteor array
2170 #Meteor array
2176 # arrayMeteors[:,0] = heiRang[arrayMeteors[:,0].astype(int)]
2171 # arrayMeteors[:,0] = heiRang[arrayMeteors[:,0].astype(int)]
2177 # arrayMeteors = numpy.hstack((arrayDate, arrayMeteors))
2172 # arrayMeteors = numpy.hstack((arrayDate, arrayMeteors))
2178
2173
2179 #Parameters Array
2174 #Parameters Array
2180 arrayParameters[:,0] = arrayDate #Date
2175 arrayParameters[:,0] = arrayDate #Date
2181 arrayParameters[:,1] = heiRang[arrayMeteors[:,0].astype(int)] #Range
2176 arrayParameters[:,1] = heiRang[arrayMeteors[:,0].astype(int)] #Range
2182 arrayParameters[:,6:8] = arrayMeteors[:,-3:-1] #Radial velocity and its error
2177 arrayParameters[:,6:8] = arrayMeteors[:,-3:-1] #Radial velocity and its error
2183 arrayParameters[:,8:12] = arrayMeteors[:,7:11] #Phases
2178 arrayParameters[:,8:12] = arrayMeteors[:,7:11] #Phases
2184 arrayParameters[:,-1] = arrayMeteors[:,-1] #Error
2179 arrayParameters[:,-1] = arrayMeteors[:,-1] #Error
2185
2180
2186
2181
2187 return arrayParameters
2182 return arrayParameters
2188
2183
2189 class CorrectSMPhases(Operation):
2184 class CorrectSMPhases(Operation):
2190
2185
2191 def run(self, dataOut, phaseOffsets, hmin = 50, hmax = 150, azimuth = 45, channelPositions = None):
2186 def run(self, dataOut, phaseOffsets, hmin = 50, hmax = 150, azimuth = 45, channelPositions = None):
2192
2187
2193 arrayParameters = dataOut.data_param
2188 arrayParameters = dataOut.data_param
2194 pairsList = []
2189 pairsList = []
2195 pairx = (0,1)
2190 pairx = (0,1)
2196 pairy = (2,3)
2191 pairy = (2,3)
2197 pairsList.append(pairx)
2192 pairsList.append(pairx)
2198 pairsList.append(pairy)
2193 pairsList.append(pairy)
2199 jph = numpy.zeros(4)
2194 jph = numpy.zeros(4)
2200
2195
2201 phaseOffsets = numpy.array(phaseOffsets)*numpy.pi/180
2196 phaseOffsets = numpy.array(phaseOffsets)*numpy.pi/180
2202 # arrayParameters[:,8:12] = numpy.unwrap(arrayParameters[:,8:12] + phaseOffsets)
2197 # arrayParameters[:,8:12] = numpy.unwrap(arrayParameters[:,8:12] + phaseOffsets)
2203 arrayParameters[:,8:12] = numpy.angle(numpy.exp(1j*(arrayParameters[:,8:12] + phaseOffsets)))
2198 arrayParameters[:,8:12] = numpy.angle(numpy.exp(1j*(arrayParameters[:,8:12] + phaseOffsets)))
2204
2199
2205 meteorOps = SMOperations()
2200 meteorOps = SMOperations()
2206 if channelPositions is None:
2201 if channelPositions is None:
2207 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
2202 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
2208 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
2203 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
2209
2204
2210 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
2205 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
2211 h = (hmin,hmax)
2206 h = (hmin,hmax)
2212
2207
2213 arrayParameters = meteorOps.getMeteorParams(arrayParameters, azimuth, h, pairsList, distances, jph)
2208 arrayParameters = meteorOps.getMeteorParams(arrayParameters, azimuth, h, pairsList, distances, jph)
2214
2209
2215 dataOut.data_param = arrayParameters
2210 dataOut.data_param = arrayParameters
2216 return
2211 return
2217
2212
2218 class SMPhaseCalibration(Operation):
2213 class SMPhaseCalibration(Operation):
2219
2214
2220 __buffer = None
2215 __buffer = None
2221
2216
2222 __initime = None
2217 __initime = None
2223
2218
2224 __dataReady = False
2219 __dataReady = False
2225
2220
2226 __isConfig = False
2221 __isConfig = False
2227
2222
2228 def __checkTime(self, currentTime, initTime, paramInterval, outputInterval):
2223 def __checkTime(self, currentTime, initTime, paramInterval, outputInterval):
2229
2224
2230 dataTime = currentTime + paramInterval
2225 dataTime = currentTime + paramInterval
2231 deltaTime = dataTime - initTime
2226 deltaTime = dataTime - initTime
2232
2227
2233 if deltaTime >= outputInterval or deltaTime < 0:
2228 if deltaTime >= outputInterval or deltaTime < 0:
2234 return True
2229 return True
2235
2230
2236 return False
2231 return False
2237
2232
2238 def __getGammas(self, pairs, d, phases):
2233 def __getGammas(self, pairs, d, phases):
2239 gammas = numpy.zeros(2)
2234 gammas = numpy.zeros(2)
2240
2235
2241 for i in range(len(pairs)):
2236 for i in range(len(pairs)):
2242
2237
2243 pairi = pairs[i]
2238 pairi = pairs[i]
2244
2239
2245 phip3 = phases[:,pairi[1]]
2240 phip3 = phases[:,pairi[1]]
2246 d3 = d[pairi[1]]
2241 d3 = d[pairi[1]]
2247 phip2 = phases[:,pairi[0]]
2242 phip2 = phases[:,pairi[0]]
2248 d2 = d[pairi[0]]
2243 d2 = d[pairi[0]]
2249 #Calculating gamma
2244 #Calculating gamma
2250 # jdcos = alp1/(k*d1)
2245 # jdcos = alp1/(k*d1)
2251 # jgamma = numpy.angle(numpy.exp(1j*(d0*alp1/d1 - alp0)))
2246 # jgamma = numpy.angle(numpy.exp(1j*(d0*alp1/d1 - alp0)))
2252 jgamma = -phip2*d3/d2 - phip3
2247 jgamma = -phip2*d3/d2 - phip3
2253 jgamma = numpy.angle(numpy.exp(1j*jgamma))
2248 jgamma = numpy.angle(numpy.exp(1j*jgamma))
2254 # jgamma[jgamma>numpy.pi] -= 2*numpy.pi
2249 # jgamma[jgamma>numpy.pi] -= 2*numpy.pi
2255 # jgamma[jgamma<-numpy.pi] += 2*numpy.pi
2250 # jgamma[jgamma<-numpy.pi] += 2*numpy.pi
2256
2251
2257 #Revised distribution
2252 #Revised distribution
2258 jgammaArray = numpy.hstack((jgamma,jgamma+0.5*numpy.pi,jgamma-0.5*numpy.pi))
2253 jgammaArray = numpy.hstack((jgamma,jgamma+0.5*numpy.pi,jgamma-0.5*numpy.pi))
2259
2254
2260 #Histogram
2255 #Histogram
2261 nBins = 64.0
2256 nBins = 64.0
2262 rmin = -0.5*numpy.pi
2257 rmin = -0.5*numpy.pi
2263 rmax = 0.5*numpy.pi
2258 rmax = 0.5*numpy.pi
2264 phaseHisto = numpy.histogram(jgammaArray, bins=nBins, range=(rmin,rmax))
2259 phaseHisto = numpy.histogram(jgammaArray, bins=nBins, range=(rmin,rmax))
2265
2260
2266 meteorsY = phaseHisto[0]
2261 meteorsY = phaseHisto[0]
2267 phasesX = phaseHisto[1][:-1]
2262 phasesX = phaseHisto[1][:-1]
2268 width = phasesX[1] - phasesX[0]
2263 width = phasesX[1] - phasesX[0]
2269 phasesX += width/2
2264 phasesX += width/2
2270
2265
2271 #Gaussian aproximation
2266 #Gaussian aproximation
2272 bpeak = meteorsY.argmax()
2267 bpeak = meteorsY.argmax()
2273 peak = meteorsY.max()
2268 peak = meteorsY.max()
2274 jmin = bpeak - 5
2269 jmin = bpeak - 5
2275 jmax = bpeak + 5 + 1
2270 jmax = bpeak + 5 + 1
2276
2271
2277 if jmin<0:
2272 if jmin<0:
2278 jmin = 0
2273 jmin = 0
2279 jmax = 6
2274 jmax = 6
2280 elif jmax > meteorsY.size:
2275 elif jmax > meteorsY.size:
2281 jmin = meteorsY.size - 6
2276 jmin = meteorsY.size - 6
2282 jmax = meteorsY.size
2277 jmax = meteorsY.size
2283
2278
2284 x0 = numpy.array([peak,bpeak,50])
2279 x0 = numpy.array([peak,bpeak,50])
2285 coeff = optimize.leastsq(self.__residualFunction, x0, args=(meteorsY[jmin:jmax], phasesX[jmin:jmax]))
2280 coeff = optimize.leastsq(self.__residualFunction, x0, args=(meteorsY[jmin:jmax], phasesX[jmin:jmax]))
2286
2281
2287 #Gammas
2282 #Gammas
2288 gammas[i] = coeff[0][1]
2283 gammas[i] = coeff[0][1]
2289
2284
2290 return gammas
2285 return gammas
2291
2286
2292 def __residualFunction(self, coeffs, y, t):
2287 def __residualFunction(self, coeffs, y, t):
2293
2288
2294 return y - self.__gauss_function(t, coeffs)
2289 return y - self.__gauss_function(t, coeffs)
2295
2290
2296 def __gauss_function(self, t, coeffs):
2291 def __gauss_function(self, t, coeffs):
2297
2292
2298 return coeffs[0]*numpy.exp(-0.5*((t - coeffs[1]) / coeffs[2])**2)
2293 return coeffs[0]*numpy.exp(-0.5*((t - coeffs[1]) / coeffs[2])**2)
2299
2294
2300 def __getPhases(self, azimuth, h, pairsList, d, gammas, meteorsArray):
2295 def __getPhases(self, azimuth, h, pairsList, d, gammas, meteorsArray):
2301 meteorOps = SMOperations()
2296 meteorOps = SMOperations()
2302 nchan = 4
2297 nchan = 4
2303 pairx = pairsList[0]
2298 pairx = pairsList[0]
2304 pairy = pairsList[1]
2299 pairy = pairsList[1]
2305 center_xangle = 0
2300 center_xangle = 0
2306 center_yangle = 0
2301 center_yangle = 0
2307 range_angle = numpy.array([10*numpy.pi,numpy.pi,numpy.pi/2,numpy.pi/4])
2302 range_angle = numpy.array([10*numpy.pi,numpy.pi,numpy.pi/2,numpy.pi/4])
2308 ntimes = len(range_angle)
2303 ntimes = len(range_angle)
2309
2304
2310 nstepsx = 20.0
2305 nstepsx = 20.0
2311 nstepsy = 20.0
2306 nstepsy = 20.0
2312
2307
2313 for iz in range(ntimes):
2308 for iz in range(ntimes):
2314 min_xangle = -range_angle[iz]/2 + center_xangle
2309 min_xangle = -range_angle[iz]/2 + center_xangle
2315 max_xangle = range_angle[iz]/2 + center_xangle
2310 max_xangle = range_angle[iz]/2 + center_xangle
2316 min_yangle = -range_angle[iz]/2 + center_yangle
2311 min_yangle = -range_angle[iz]/2 + center_yangle
2317 max_yangle = range_angle[iz]/2 + center_yangle
2312 max_yangle = range_angle[iz]/2 + center_yangle
2318
2313
2319 inc_x = (max_xangle-min_xangle)/nstepsx
2314 inc_x = (max_xangle-min_xangle)/nstepsx
2320 inc_y = (max_yangle-min_yangle)/nstepsy
2315 inc_y = (max_yangle-min_yangle)/nstepsy
2321
2316
2322 alpha_y = numpy.arange(nstepsy)*inc_y + min_yangle
2317 alpha_y = numpy.arange(nstepsy)*inc_y + min_yangle
2323 alpha_x = numpy.arange(nstepsx)*inc_x + min_xangle
2318 alpha_x = numpy.arange(nstepsx)*inc_x + min_xangle
2324 penalty = numpy.zeros((nstepsx,nstepsy))
2319 penalty = numpy.zeros((nstepsx,nstepsy))
2325 jph_array = numpy.zeros((nchan,nstepsx,nstepsy))
2320 jph_array = numpy.zeros((nchan,nstepsx,nstepsy))
2326 jph = numpy.zeros(nchan)
2321 jph = numpy.zeros(nchan)
2327
2322
2328 # Iterations looking for the offset
2323 # Iterations looking for the offset
2329 for iy in range(int(nstepsy)):
2324 for iy in range(int(nstepsy)):
2330 for ix in range(int(nstepsx)):
2325 for ix in range(int(nstepsx)):
2331 jph[pairy[1]] = alpha_y[iy]
2326 jph[pairy[1]] = alpha_y[iy]
2332 jph[pairy[0]] = -gammas[1] - alpha_y[iy]*d[pairy[1]]/d[pairy[0]]
2327 jph[pairy[0]] = -gammas[1] - alpha_y[iy]*d[pairy[1]]/d[pairy[0]]
2333
2328
2334 jph[pairx[1]] = alpha_x[ix]
2329 jph[pairx[1]] = alpha_x[ix]
2335 jph[pairx[0]] = -gammas[0] - alpha_x[ix]*d[pairx[1]]/d[pairx[0]]
2330 jph[pairx[0]] = -gammas[0] - alpha_x[ix]*d[pairx[1]]/d[pairx[0]]
2336
2331
2337 jph_array[:,ix,iy] = jph
2332 jph_array[:,ix,iy] = jph
2338
2333
2339 meteorsArray1 = meteorOps.getMeteorParams(meteorsArray, azimuth, h, pairsList, d, jph)
2334 meteorsArray1 = meteorOps.getMeteorParams(meteorsArray, azimuth, h, pairsList, d, jph)
2340 error = meteorsArray1[:,-1]
2335 error = meteorsArray1[:,-1]
2341 ind1 = numpy.where(error==0)[0]
2336 ind1 = numpy.where(error==0)[0]
2342 penalty[ix,iy] = ind1.size
2337 penalty[ix,iy] = ind1.size
2343
2338
2344 i,j = numpy.unravel_index(penalty.argmax(), penalty.shape)
2339 i,j = numpy.unravel_index(penalty.argmax(), penalty.shape)
2345 phOffset = jph_array[:,i,j]
2340 phOffset = jph_array[:,i,j]
2346
2341
2347 center_xangle = phOffset[pairx[1]]
2342 center_xangle = phOffset[pairx[1]]
2348 center_yangle = phOffset[pairy[1]]
2343 center_yangle = phOffset[pairy[1]]
2349
2344
2350 phOffset = numpy.angle(numpy.exp(1j*jph_array[:,i,j]))
2345 phOffset = numpy.angle(numpy.exp(1j*jph_array[:,i,j]))
2351 phOffset = phOffset*180/numpy.pi
2346 phOffset = phOffset*180/numpy.pi
2352 return phOffset
2347 return phOffset
2353
2348
2354
2349
2355 def run(self, dataOut, hmin, hmax, channelPositions=None, nHours = 1):
2350 def run(self, dataOut, hmin, hmax, channelPositions=None, nHours = 1):
2356
2351
2357 dataOut.flagNoData = True
2352 dataOut.flagNoData = True
2358 self.__dataReady = False
2353 self.__dataReady = False
2359 dataOut.outputInterval = nHours*3600
2354 dataOut.outputInterval = nHours*3600
2360
2355
2361 if self.__isConfig == False:
2356 if self.__isConfig == False:
2362 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
2357 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
2363 #Get Initial LTC time
2358 #Get Initial LTC time
2364 self.__initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
2359 self.__initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
2365 self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
2360 self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
2366
2361
2367 self.__isConfig = True
2362 self.__isConfig = True
2368
2363
2369 if self.__buffer is None:
2364 if self.__buffer is None:
2370 self.__buffer = dataOut.data_param.copy()
2365 self.__buffer = dataOut.data_param.copy()
2371
2366
2372 else:
2367 else:
2373 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
2368 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
2374
2369
2375 self.__dataReady = self.__checkTime(dataOut.utctime, self.__initime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
2370 self.__dataReady = self.__checkTime(dataOut.utctime, self.__initime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
2376
2371
2377 if self.__dataReady:
2372 if self.__dataReady:
2378 dataOut.utctimeInit = self.__initime
2373 dataOut.utctimeInit = self.__initime
2379 self.__initime += dataOut.outputInterval #to erase time offset
2374 self.__initime += dataOut.outputInterval #to erase time offset
2380
2375
2381 freq = dataOut.frequency
2376 freq = dataOut.frequency
2382 c = dataOut.C #m/s
2377 c = dataOut.C #m/s
2383 lamb = c/freq
2378 lamb = c/freq
2384 k = 2*numpy.pi/lamb
2379 k = 2*numpy.pi/lamb
2385 azimuth = 0
2380 azimuth = 0
2386 h = (hmin, hmax)
2381 h = (hmin, hmax)
2387 pairs = ((0,1),(2,3))
2382 pairs = ((0,1),(2,3))
2388
2383
2389 if channelPositions is None:
2384 if channelPositions is None:
2390 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
2385 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
2391 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
2386 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
2392 meteorOps = SMOperations()
2387 meteorOps = SMOperations()
2393 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
2388 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
2394
2389
2395 # distances1 = [-distances[0]*lamb, distances[1]*lamb, -distances[2]*lamb, distances[3]*lamb]
2390 # distances1 = [-distances[0]*lamb, distances[1]*lamb, -distances[2]*lamb, distances[3]*lamb]
2396
2391
2397 meteorsArray = self.__buffer
2392 meteorsArray = self.__buffer
2398 error = meteorsArray[:,-1]
2393 error = meteorsArray[:,-1]
2399 boolError = (error==0)|(error==3)|(error==4)|(error==13)|(error==14)
2394 boolError = (error==0)|(error==3)|(error==4)|(error==13)|(error==14)
2400 ind1 = numpy.where(boolError)[0]
2395 ind1 = numpy.where(boolError)[0]
2401 meteorsArray = meteorsArray[ind1,:]
2396 meteorsArray = meteorsArray[ind1,:]
2402 meteorsArray[:,-1] = 0
2397 meteorsArray[:,-1] = 0
2403 phases = meteorsArray[:,8:12]
2398 phases = meteorsArray[:,8:12]
2404
2399
2405 #Calculate Gammas
2400 #Calculate Gammas
2406 gammas = self.__getGammas(pairs, distances, phases)
2401 gammas = self.__getGammas(pairs, distances, phases)
2407 # gammas = numpy.array([-21.70409463,45.76935864])*numpy.pi/180
2402 # gammas = numpy.array([-21.70409463,45.76935864])*numpy.pi/180
2408 #Calculate Phases
2403 #Calculate Phases
2409 phasesOff = self.__getPhases(azimuth, h, pairs, distances, gammas, meteorsArray)
2404 phasesOff = self.__getPhases(azimuth, h, pairs, distances, gammas, meteorsArray)
2410 phasesOff = phasesOff.reshape((1,phasesOff.size))
2405 phasesOff = phasesOff.reshape((1,phasesOff.size))
2411 dataOut.data_output = -phasesOff
2406 dataOut.data_output = -phasesOff
2412 dataOut.flagNoData = False
2407 dataOut.flagNoData = False
2413 dataOut.channelList = pairslist0
2408 dataOut.channelList = pairslist0
2414 self.__buffer = None
2409 self.__buffer = None
2415
2410
2416
2411
2417 return
2412 return
2418
2413
2419 class SMOperations():
2414 class SMOperations():
2420
2415
2421 def __init__(self):
2416 def __init__(self):
2422
2417
2423 return
2418 return
2424
2419
2425 def getMeteorParams(self, arrayParameters0, azimuth, h, pairsList, distances, jph):
2420 def getMeteorParams(self, arrayParameters0, azimuth, h, pairsList, distances, jph):
2426
2421
2427 arrayParameters = arrayParameters0.copy()
2422 arrayParameters = arrayParameters0.copy()
2428 hmin = h[0]
2423 hmin = h[0]
2429 hmax = h[1]
2424 hmax = h[1]
2430
2425
2431 #Calculate AOA (Error N 3, 4)
2426 #Calculate AOA (Error N 3, 4)
2432 #JONES ET AL. 1998
2427 #JONES ET AL. 1998
2433 AOAthresh = numpy.pi/8
2428 AOAthresh = numpy.pi/8
2434 error = arrayParameters[:,-1]
2429 error = arrayParameters[:,-1]
2435 phases = -arrayParameters[:,8:12] + jph
2430 phases = -arrayParameters[:,8:12] + jph
2436 # phases = numpy.unwrap(phases)
2431 # phases = numpy.unwrap(phases)
2437 arrayParameters[:,3:6], arrayParameters[:,-1] = self.__getAOA(phases, pairsList, distances, error, AOAthresh, azimuth)
2432 arrayParameters[:,3:6], arrayParameters[:,-1] = self.__getAOA(phases, pairsList, distances, error, AOAthresh, azimuth)
2438
2433
2439 #Calculate Heights (Error N 13 and 14)
2434 #Calculate Heights (Error N 13 and 14)
2440 error = arrayParameters[:,-1]
2435 error = arrayParameters[:,-1]
2441 Ranges = arrayParameters[:,1]
2436 Ranges = arrayParameters[:,1]
2442 zenith = arrayParameters[:,4]
2437 zenith = arrayParameters[:,4]
2443 arrayParameters[:,2], arrayParameters[:,-1] = self.__getHeights(Ranges, zenith, error, hmin, hmax)
2438 arrayParameters[:,2], arrayParameters[:,-1] = self.__getHeights(Ranges, zenith, error, hmin, hmax)
2444
2439
2445 #----------------------- Get Final data ------------------------------------
2440 #----------------------- Get Final data ------------------------------------
2446 # error = arrayParameters[:,-1]
2441 # error = arrayParameters[:,-1]
2447 # ind1 = numpy.where(error==0)[0]
2442 # ind1 = numpy.where(error==0)[0]
2448 # arrayParameters = arrayParameters[ind1,:]
2443 # arrayParameters = arrayParameters[ind1,:]
2449
2444
2450 return arrayParameters
2445 return arrayParameters
2451
2446
2452 def __getAOA(self, phases, pairsList, directions, error, AOAthresh, azimuth):
2447 def __getAOA(self, phases, pairsList, directions, error, AOAthresh, azimuth):
2453
2448
2454 arrayAOA = numpy.zeros((phases.shape[0],3))
2449 arrayAOA = numpy.zeros((phases.shape[0],3))
2455 cosdir0, cosdir = self.__getDirectionCosines(phases, pairsList,directions)
2450 cosdir0, cosdir = self.__getDirectionCosines(phases, pairsList,directions)
2456
2451
2457 arrayAOA[:,:2] = self.__calculateAOA(cosdir, azimuth)
2452 arrayAOA[:,:2] = self.__calculateAOA(cosdir, azimuth)
2458 cosDirError = numpy.sum(numpy.abs(cosdir0 - cosdir), axis = 1)
2453 cosDirError = numpy.sum(numpy.abs(cosdir0 - cosdir), axis = 1)
2459 arrayAOA[:,2] = cosDirError
2454 arrayAOA[:,2] = cosDirError
2460
2455
2461 azimuthAngle = arrayAOA[:,0]
2456 azimuthAngle = arrayAOA[:,0]
2462 zenithAngle = arrayAOA[:,1]
2457 zenithAngle = arrayAOA[:,1]
2463
2458
2464 #Setting Error
2459 #Setting Error
2465 indError = numpy.where(numpy.logical_or(error == 3, error == 4))[0]
2460 indError = numpy.where(numpy.logical_or(error == 3, error == 4))[0]
2466 error[indError] = 0
2461 error[indError] = 0
2467 #Number 3: AOA not fesible
2462 #Number 3: AOA not fesible
2468 indInvalid = numpy.where(numpy.logical_and((numpy.logical_or(numpy.isnan(zenithAngle), numpy.isnan(azimuthAngle))),error == 0))[0]
2463 indInvalid = numpy.where(numpy.logical_and((numpy.logical_or(numpy.isnan(zenithAngle), numpy.isnan(azimuthAngle))),error == 0))[0]
2469 error[indInvalid] = 3
2464 error[indInvalid] = 3
2470 #Number 4: Large difference in AOAs obtained from different antenna baselines
2465 #Number 4: Large difference in AOAs obtained from different antenna baselines
2471 indInvalid = numpy.where(numpy.logical_and(cosDirError > AOAthresh,error == 0))[0]
2466 indInvalid = numpy.where(numpy.logical_and(cosDirError > AOAthresh,error == 0))[0]
2472 error[indInvalid] = 4
2467 error[indInvalid] = 4
2473 return arrayAOA, error
2468 return arrayAOA, error
2474
2469
2475 def __getDirectionCosines(self, arrayPhase, pairsList, distances):
2470 def __getDirectionCosines(self, arrayPhase, pairsList, distances):
2476
2471
2477 #Initializing some variables
2472 #Initializing some variables
2478 ang_aux = numpy.array([-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8])*2*numpy.pi
2473 ang_aux = numpy.array([-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8])*2*numpy.pi
2479 ang_aux = ang_aux.reshape(1,ang_aux.size)
2474 ang_aux = ang_aux.reshape(1,ang_aux.size)
2480
2475
2481 cosdir = numpy.zeros((arrayPhase.shape[0],2))
2476 cosdir = numpy.zeros((arrayPhase.shape[0],2))
2482 cosdir0 = numpy.zeros((arrayPhase.shape[0],2))
2477 cosdir0 = numpy.zeros((arrayPhase.shape[0],2))
2483
2478
2484
2479
2485 for i in range(2):
2480 for i in range(2):
2486 ph0 = arrayPhase[:,pairsList[i][0]]
2481 ph0 = arrayPhase[:,pairsList[i][0]]
2487 ph1 = arrayPhase[:,pairsList[i][1]]
2482 ph1 = arrayPhase[:,pairsList[i][1]]
2488 d0 = distances[pairsList[i][0]]
2483 d0 = distances[pairsList[i][0]]
2489 d1 = distances[pairsList[i][1]]
2484 d1 = distances[pairsList[i][1]]
2490
2485
2491 ph0_aux = ph0 + ph1
2486 ph0_aux = ph0 + ph1
2492 ph0_aux = numpy.angle(numpy.exp(1j*ph0_aux))
2487 ph0_aux = numpy.angle(numpy.exp(1j*ph0_aux))
2493 # ph0_aux[ph0_aux > numpy.pi] -= 2*numpy.pi
2488 # ph0_aux[ph0_aux > numpy.pi] -= 2*numpy.pi
2494 # ph0_aux[ph0_aux < -numpy.pi] += 2*numpy.pi
2489 # ph0_aux[ph0_aux < -numpy.pi] += 2*numpy.pi
2495 #First Estimation
2490 #First Estimation
2496 cosdir0[:,i] = (ph0_aux)/(2*numpy.pi*(d0 - d1))
2491 cosdir0[:,i] = (ph0_aux)/(2*numpy.pi*(d0 - d1))
2497
2492
2498 #Most-Accurate Second Estimation
2493 #Most-Accurate Second Estimation
2499 phi1_aux = ph0 - ph1
2494 phi1_aux = ph0 - ph1
2500 phi1_aux = phi1_aux.reshape(phi1_aux.size,1)
2495 phi1_aux = phi1_aux.reshape(phi1_aux.size,1)
2501 #Direction Cosine 1
2496 #Direction Cosine 1
2502 cosdir1 = (phi1_aux + ang_aux)/(2*numpy.pi*(d0 + d1))
2497 cosdir1 = (phi1_aux + ang_aux)/(2*numpy.pi*(d0 + d1))
2503
2498
2504 #Searching the correct Direction Cosine
2499 #Searching the correct Direction Cosine
2505 cosdir0_aux = cosdir0[:,i]
2500 cosdir0_aux = cosdir0[:,i]
2506 cosdir0_aux = cosdir0_aux.reshape(cosdir0_aux.size,1)
2501 cosdir0_aux = cosdir0_aux.reshape(cosdir0_aux.size,1)
2507 #Minimum Distance
2502 #Minimum Distance
2508 cosDiff = (cosdir1 - cosdir0_aux)**2
2503 cosDiff = (cosdir1 - cosdir0_aux)**2
2509 indcos = cosDiff.argmin(axis = 1)
2504 indcos = cosDiff.argmin(axis = 1)
2510 #Saving Value obtained
2505 #Saving Value obtained
2511 cosdir[:,i] = cosdir1[numpy.arange(len(indcos)),indcos]
2506 cosdir[:,i] = cosdir1[numpy.arange(len(indcos)),indcos]
2512
2507
2513 return cosdir0, cosdir
2508 return cosdir0, cosdir
2514
2509
2515 def __calculateAOA(self, cosdir, azimuth):
2510 def __calculateAOA(self, cosdir, azimuth):
2516 cosdirX = cosdir[:,0]
2511 cosdirX = cosdir[:,0]
2517 cosdirY = cosdir[:,1]
2512 cosdirY = cosdir[:,1]
2518
2513
2519 zenithAngle = numpy.arccos(numpy.sqrt(1 - cosdirX**2 - cosdirY**2))*180/numpy.pi
2514 zenithAngle = numpy.arccos(numpy.sqrt(1 - cosdirX**2 - cosdirY**2))*180/numpy.pi
2520 azimuthAngle = numpy.arctan2(cosdirX,cosdirY)*180/numpy.pi + azimuth#0 deg north, 90 deg east
2515 azimuthAngle = numpy.arctan2(cosdirX,cosdirY)*180/numpy.pi + azimuth#0 deg north, 90 deg east
2521 angles = numpy.vstack((azimuthAngle, zenithAngle)).transpose()
2516 angles = numpy.vstack((azimuthAngle, zenithAngle)).transpose()
2522
2517
2523 return angles
2518 return angles
2524
2519
2525 def __getHeights(self, Ranges, zenith, error, minHeight, maxHeight):
2520 def __getHeights(self, Ranges, zenith, error, minHeight, maxHeight):
2526
2521
2527 Ramb = 375 #Ramb = c/(2*PRF)
2522 Ramb = 375 #Ramb = c/(2*PRF)
2528 Re = 6371 #Earth Radius
2523 Re = 6371 #Earth Radius
2529 heights = numpy.zeros(Ranges.shape)
2524 heights = numpy.zeros(Ranges.shape)
2530
2525
2531 R_aux = numpy.array([0,1,2])*Ramb
2526 R_aux = numpy.array([0,1,2])*Ramb
2532 R_aux = R_aux.reshape(1,R_aux.size)
2527 R_aux = R_aux.reshape(1,R_aux.size)
2533
2528
2534 Ranges = Ranges.reshape(Ranges.size,1)
2529 Ranges = Ranges.reshape(Ranges.size,1)
2535
2530
2536 Ri = Ranges + R_aux
2531 Ri = Ranges + R_aux
2537 hi = numpy.sqrt(Re**2 + Ri**2 + (2*Re*numpy.cos(zenith*numpy.pi/180)*Ri.transpose()).transpose()) - Re
2532 hi = numpy.sqrt(Re**2 + Ri**2 + (2*Re*numpy.cos(zenith*numpy.pi/180)*Ri.transpose()).transpose()) - Re
2538
2533
2539 #Check if there is a height between 70 and 110 km
2534 #Check if there is a height between 70 and 110 km
2540 h_bool = numpy.sum(numpy.logical_and(hi > minHeight, hi < maxHeight), axis = 1)
2535 h_bool = numpy.sum(numpy.logical_and(hi > minHeight, hi < maxHeight), axis = 1)
2541 ind_h = numpy.where(h_bool == 1)[0]
2536 ind_h = numpy.where(h_bool == 1)[0]
2542
2537
2543 hCorr = hi[ind_h, :]
2538 hCorr = hi[ind_h, :]
2544 ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight))
2539 ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight))
2545
2540
2546 hCorr = hi[ind_hCorr]
2541 hCorr = hi[ind_hCorr]
2547 heights[ind_h] = hCorr
2542 heights[ind_h] = hCorr
2548
2543
2549 #Setting Error
2544 #Setting Error
2550 #Number 13: Height unresolvable echo: not valid height within 70 to 110 km
2545 #Number 13: Height unresolvable echo: not valid height within 70 to 110 km
2551 #Number 14: Height ambiguous echo: more than one possible height within 70 to 110 km
2546 #Number 14: Height ambiguous echo: more than one possible height within 70 to 110 km
2552 indError = numpy.where(numpy.logical_or(error == 13, error == 14))[0]
2547 indError = numpy.where(numpy.logical_or(error == 13, error == 14))[0]
2553 error[indError] = 0
2548 error[indError] = 0
2554 indInvalid2 = numpy.where(numpy.logical_and(h_bool > 1, error == 0))[0]
2549 indInvalid2 = numpy.where(numpy.logical_and(h_bool > 1, error == 0))[0]
2555 error[indInvalid2] = 14
2550 error[indInvalid2] = 14
2556 indInvalid1 = numpy.where(numpy.logical_and(h_bool == 0, error == 0))[0]
2551 indInvalid1 = numpy.where(numpy.logical_and(h_bool == 0, error == 0))[0]
2557 error[indInvalid1] = 13
2552 error[indInvalid1] = 13
2558
2553
2559 return heights, error
2554 return heights, error
2560
2555
2561 def getPhasePairs(self, channelPositions):
2556 def getPhasePairs(self, channelPositions):
2562 chanPos = numpy.array(channelPositions)
2557 chanPos = numpy.array(channelPositions)
2563 listOper = list(itertools.combinations(range(5),2))
2558 listOper = list(itertools.combinations(range(5),2))
2564
2559
2565 distances = numpy.zeros(4)
2560 distances = numpy.zeros(4)
2566 axisX = []
2561 axisX = []
2567 axisY = []
2562 axisY = []
2568 distX = numpy.zeros(3)
2563 distX = numpy.zeros(3)
2569 distY = numpy.zeros(3)
2564 distY = numpy.zeros(3)
2570 ix = 0
2565 ix = 0
2571 iy = 0
2566 iy = 0
2572
2567
2573 pairX = numpy.zeros((2,2))
2568 pairX = numpy.zeros((2,2))
2574 pairY = numpy.zeros((2,2))
2569 pairY = numpy.zeros((2,2))
2575
2570
2576 for i in range(len(listOper)):
2571 for i in range(len(listOper)):
2577 pairi = listOper[i]
2572 pairi = listOper[i]
2578
2573
2579 posDif = numpy.abs(chanPos[pairi[0],:] - chanPos[pairi[1],:])
2574 posDif = numpy.abs(chanPos[pairi[0],:] - chanPos[pairi[1],:])
2580
2575
2581 if posDif[0] == 0:
2576 if posDif[0] == 0:
2582 axisY.append(pairi)
2577 axisY.append(pairi)
2583 distY[iy] = posDif[1]
2578 distY[iy] = posDif[1]
2584 iy += 1
2579 iy += 1
2585 elif posDif[1] == 0:
2580 elif posDif[1] == 0:
2586 axisX.append(pairi)
2581 axisX.append(pairi)
2587 distX[ix] = posDif[0]
2582 distX[ix] = posDif[0]
2588 ix += 1
2583 ix += 1
2589
2584
2590 for i in range(2):
2585 for i in range(2):
2591 if i==0:
2586 if i==0:
2592 dist0 = distX
2587 dist0 = distX
2593 axis0 = axisX
2588 axis0 = axisX
2594 else:
2589 else:
2595 dist0 = distY
2590 dist0 = distY
2596 axis0 = axisY
2591 axis0 = axisY
2597
2592
2598 side = numpy.argsort(dist0)[:-1]
2593 side = numpy.argsort(dist0)[:-1]
2599 axis0 = numpy.array(axis0)[side,:]
2594 axis0 = numpy.array(axis0)[side,:]
2600 chanC = int(numpy.intersect1d(axis0[0,:], axis0[1,:])[0])
2595 chanC = int(numpy.intersect1d(axis0[0,:], axis0[1,:])[0])
2601 axis1 = numpy.unique(numpy.reshape(axis0,4))
2596 axis1 = numpy.unique(numpy.reshape(axis0,4))
2602 side = axis1[axis1 != chanC]
2597 side = axis1[axis1 != chanC]
2603 diff1 = chanPos[chanC,i] - chanPos[side[0],i]
2598 diff1 = chanPos[chanC,i] - chanPos[side[0],i]
2604 diff2 = chanPos[chanC,i] - chanPos[side[1],i]
2599 diff2 = chanPos[chanC,i] - chanPos[side[1],i]
2605 if diff1<0:
2600 if diff1<0:
2606 chan2 = side[0]
2601 chan2 = side[0]
2607 d2 = numpy.abs(diff1)
2602 d2 = numpy.abs(diff1)
2608 chan1 = side[1]
2603 chan1 = side[1]
2609 d1 = numpy.abs(diff2)
2604 d1 = numpy.abs(diff2)
2610 else:
2605 else:
2611 chan2 = side[1]
2606 chan2 = side[1]
2612 d2 = numpy.abs(diff2)
2607 d2 = numpy.abs(diff2)
2613 chan1 = side[0]
2608 chan1 = side[0]
2614 d1 = numpy.abs(diff1)
2609 d1 = numpy.abs(diff1)
2615
2610
2616 if i==0:
2611 if i==0:
2617 chanCX = chanC
2612 chanCX = chanC
2618 chan1X = chan1
2613 chan1X = chan1
2619 chan2X = chan2
2614 chan2X = chan2
2620 distances[0:2] = numpy.array([d1,d2])
2615 distances[0:2] = numpy.array([d1,d2])
2621 else:
2616 else:
2622 chanCY = chanC
2617 chanCY = chanC
2623 chan1Y = chan1
2618 chan1Y = chan1
2624 chan2Y = chan2
2619 chan2Y = chan2
2625 distances[2:4] = numpy.array([d1,d2])
2620 distances[2:4] = numpy.array([d1,d2])
2626 # axisXsides = numpy.reshape(axisX[ix,:],4)
2621 # axisXsides = numpy.reshape(axisX[ix,:],4)
2627 #
2622 #
2628 # channelCentX = int(numpy.intersect1d(pairX[0,:], pairX[1,:])[0])
2623 # channelCentX = int(numpy.intersect1d(pairX[0,:], pairX[1,:])[0])
2629 # channelCentY = int(numpy.intersect1d(pairY[0,:], pairY[1,:])[0])
2624 # channelCentY = int(numpy.intersect1d(pairY[0,:], pairY[1,:])[0])
2630 #
2625 #
2631 # ind25X = numpy.where(pairX[0,:] != channelCentX)[0][0]
2626 # ind25X = numpy.where(pairX[0,:] != channelCentX)[0][0]
2632 # ind20X = numpy.where(pairX[1,:] != channelCentX)[0][0]
2627 # ind20X = numpy.where(pairX[1,:] != channelCentX)[0][0]
2633 # channel25X = int(pairX[0,ind25X])
2628 # channel25X = int(pairX[0,ind25X])
2634 # channel20X = int(pairX[1,ind20X])
2629 # channel20X = int(pairX[1,ind20X])
2635 # ind25Y = numpy.where(pairY[0,:] != channelCentY)[0][0]
2630 # ind25Y = numpy.where(pairY[0,:] != channelCentY)[0][0]
2636 # ind20Y = numpy.where(pairY[1,:] != channelCentY)[0][0]
2631 # ind20Y = numpy.where(pairY[1,:] != channelCentY)[0][0]
2637 # channel25Y = int(pairY[0,ind25Y])
2632 # channel25Y = int(pairY[0,ind25Y])
2638 # channel20Y = int(pairY[1,ind20Y])
2633 # channel20Y = int(pairY[1,ind20Y])
2639
2634
2640 # pairslist = [(channelCentX, channel25X),(channelCentX, channel20X),(channelCentY,channel25Y),(channelCentY, channel20Y)]
2635 # pairslist = [(channelCentX, channel25X),(channelCentX, channel20X),(channelCentY,channel25Y),(channelCentY, channel20Y)]
2641 pairslist = [(chanCX, chan1X),(chanCX, chan2X),(chanCY,chan1Y),(chanCY, chan2Y)]
2636 pairslist = [(chanCX, chan1X),(chanCX, chan2X),(chanCY,chan1Y),(chanCY, chan2Y)]
2642
2637
2643 return pairslist, distances
2638 return pairslist, distances
2644 # def __getAOA(self, phases, pairsList, error, AOAthresh, azimuth):
2639 # def __getAOA(self, phases, pairsList, error, AOAthresh, azimuth):
2645 #
2640 #
2646 # arrayAOA = numpy.zeros((phases.shape[0],3))
2641 # arrayAOA = numpy.zeros((phases.shape[0],3))
2647 # cosdir0, cosdir = self.__getDirectionCosines(phases, pairsList)
2642 # cosdir0, cosdir = self.__getDirectionCosines(phases, pairsList)
2648 #
2643 #
2649 # arrayAOA[:,:2] = self.__calculateAOA(cosdir, azimuth)
2644 # arrayAOA[:,:2] = self.__calculateAOA(cosdir, azimuth)
2650 # cosDirError = numpy.sum(numpy.abs(cosdir0 - cosdir), axis = 1)
2645 # cosDirError = numpy.sum(numpy.abs(cosdir0 - cosdir), axis = 1)
2651 # arrayAOA[:,2] = cosDirError
2646 # arrayAOA[:,2] = cosDirError
2652 #
2647 #
2653 # azimuthAngle = arrayAOA[:,0]
2648 # azimuthAngle = arrayAOA[:,0]
2654 # zenithAngle = arrayAOA[:,1]
2649 # zenithAngle = arrayAOA[:,1]
2655 #
2650 #
2656 # #Setting Error
2651 # #Setting Error
2657 # #Number 3: AOA not fesible
2652 # #Number 3: AOA not fesible
2658 # indInvalid = numpy.where(numpy.logical_and((numpy.logical_or(numpy.isnan(zenithAngle), numpy.isnan(azimuthAngle))),error == 0))[0]
2653 # indInvalid = numpy.where(numpy.logical_and((numpy.logical_or(numpy.isnan(zenithAngle), numpy.isnan(azimuthAngle))),error == 0))[0]
2659 # error[indInvalid] = 3
2654 # error[indInvalid] = 3
2660 # #Number 4: Large difference in AOAs obtained from different antenna baselines
2655 # #Number 4: Large difference in AOAs obtained from different antenna baselines
2661 # indInvalid = numpy.where(numpy.logical_and(cosDirError > AOAthresh,error == 0))[0]
2656 # indInvalid = numpy.where(numpy.logical_and(cosDirError > AOAthresh,error == 0))[0]
2662 # error[indInvalid] = 4
2657 # error[indInvalid] = 4
2663 # return arrayAOA, error
2658 # return arrayAOA, error
2664 #
2659 #
2665 # def __getDirectionCosines(self, arrayPhase, pairsList):
2660 # def __getDirectionCosines(self, arrayPhase, pairsList):
2666 #
2661 #
2667 # #Initializing some variables
2662 # #Initializing some variables
2668 # ang_aux = numpy.array([-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8])*2*numpy.pi
2663 # ang_aux = numpy.array([-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8])*2*numpy.pi
2669 # ang_aux = ang_aux.reshape(1,ang_aux.size)
2664 # ang_aux = ang_aux.reshape(1,ang_aux.size)
2670 #
2665 #
2671 # cosdir = numpy.zeros((arrayPhase.shape[0],2))
2666 # cosdir = numpy.zeros((arrayPhase.shape[0],2))
2672 # cosdir0 = numpy.zeros((arrayPhase.shape[0],2))
2667 # cosdir0 = numpy.zeros((arrayPhase.shape[0],2))
2673 #
2668 #
2674 #
2669 #
2675 # for i in range(2):
2670 # for i in range(2):
2676 # #First Estimation
2671 # #First Estimation
2677 # phi0_aux = arrayPhase[:,pairsList[i][0]] + arrayPhase[:,pairsList[i][1]]
2672 # phi0_aux = arrayPhase[:,pairsList[i][0]] + arrayPhase[:,pairsList[i][1]]
2678 # #Dealias
2673 # #Dealias
2679 # indcsi = numpy.where(phi0_aux > numpy.pi)
2674 # indcsi = numpy.where(phi0_aux > numpy.pi)
2680 # phi0_aux[indcsi] -= 2*numpy.pi
2675 # phi0_aux[indcsi] -= 2*numpy.pi
2681 # indcsi = numpy.where(phi0_aux < -numpy.pi)
2676 # indcsi = numpy.where(phi0_aux < -numpy.pi)
2682 # phi0_aux[indcsi] += 2*numpy.pi
2677 # phi0_aux[indcsi] += 2*numpy.pi
2683 # #Direction Cosine 0
2678 # #Direction Cosine 0
2684 # cosdir0[:,i] = -(phi0_aux)/(2*numpy.pi*0.5)
2679 # cosdir0[:,i] = -(phi0_aux)/(2*numpy.pi*0.5)
2685 #
2680 #
2686 # #Most-Accurate Second Estimation
2681 # #Most-Accurate Second Estimation
2687 # phi1_aux = arrayPhase[:,pairsList[i][0]] - arrayPhase[:,pairsList[i][1]]
2682 # phi1_aux = arrayPhase[:,pairsList[i][0]] - arrayPhase[:,pairsList[i][1]]
2688 # phi1_aux = phi1_aux.reshape(phi1_aux.size,1)
2683 # phi1_aux = phi1_aux.reshape(phi1_aux.size,1)
2689 # #Direction Cosine 1
2684 # #Direction Cosine 1
2690 # cosdir1 = -(phi1_aux + ang_aux)/(2*numpy.pi*4.5)
2685 # cosdir1 = -(phi1_aux + ang_aux)/(2*numpy.pi*4.5)
2691 #
2686 #
2692 # #Searching the correct Direction Cosine
2687 # #Searching the correct Direction Cosine
2693 # cosdir0_aux = cosdir0[:,i]
2688 # cosdir0_aux = cosdir0[:,i]
2694 # cosdir0_aux = cosdir0_aux.reshape(cosdir0_aux.size,1)
2689 # cosdir0_aux = cosdir0_aux.reshape(cosdir0_aux.size,1)
2695 # #Minimum Distance
2690 # #Minimum Distance
2696 # cosDiff = (cosdir1 - cosdir0_aux)**2
2691 # cosDiff = (cosdir1 - cosdir0_aux)**2
2697 # indcos = cosDiff.argmin(axis = 1)
2692 # indcos = cosDiff.argmin(axis = 1)
2698 # #Saving Value obtained
2693 # #Saving Value obtained
2699 # cosdir[:,i] = cosdir1[numpy.arange(len(indcos)),indcos]
2694 # cosdir[:,i] = cosdir1[numpy.arange(len(indcos)),indcos]
2700 #
2695 #
2701 # return cosdir0, cosdir
2696 # return cosdir0, cosdir
2702 #
2697 #
2703 # def __calculateAOA(self, cosdir, azimuth):
2698 # def __calculateAOA(self, cosdir, azimuth):
2704 # cosdirX = cosdir[:,0]
2699 # cosdirX = cosdir[:,0]
2705 # cosdirY = cosdir[:,1]
2700 # cosdirY = cosdir[:,1]
2706 #
2701 #
2707 # zenithAngle = numpy.arccos(numpy.sqrt(1 - cosdirX**2 - cosdirY**2))*180/numpy.pi
2702 # zenithAngle = numpy.arccos(numpy.sqrt(1 - cosdirX**2 - cosdirY**2))*180/numpy.pi
2708 # azimuthAngle = numpy.arctan2(cosdirX,cosdirY)*180/numpy.pi + azimuth #0 deg north, 90 deg east
2703 # azimuthAngle = numpy.arctan2(cosdirX,cosdirY)*180/numpy.pi + azimuth #0 deg north, 90 deg east
2709 # angles = numpy.vstack((azimuthAngle, zenithAngle)).transpose()
2704 # angles = numpy.vstack((azimuthAngle, zenithAngle)).transpose()
2710 #
2705 #
2711 # return angles
2706 # return angles
2712 #
2707 #
2713 # def __getHeights(self, Ranges, zenith, error, minHeight, maxHeight):
2708 # def __getHeights(self, Ranges, zenith, error, minHeight, maxHeight):
2714 #
2709 #
2715 # Ramb = 375 #Ramb = c/(2*PRF)
2710 # Ramb = 375 #Ramb = c/(2*PRF)
2716 # Re = 6371 #Earth Radius
2711 # Re = 6371 #Earth Radius
2717 # heights = numpy.zeros(Ranges.shape)
2712 # heights = numpy.zeros(Ranges.shape)
2718 #
2713 #
2719 # R_aux = numpy.array([0,1,2])*Ramb
2714 # R_aux = numpy.array([0,1,2])*Ramb
2720 # R_aux = R_aux.reshape(1,R_aux.size)
2715 # R_aux = R_aux.reshape(1,R_aux.size)
2721 #
2716 #
2722 # Ranges = Ranges.reshape(Ranges.size,1)
2717 # Ranges = Ranges.reshape(Ranges.size,1)
2723 #
2718 #
2724 # Ri = Ranges + R_aux
2719 # Ri = Ranges + R_aux
2725 # hi = numpy.sqrt(Re**2 + Ri**2 + (2*Re*numpy.cos(zenith*numpy.pi/180)*Ri.transpose()).transpose()) - Re
2720 # hi = numpy.sqrt(Re**2 + Ri**2 + (2*Re*numpy.cos(zenith*numpy.pi/180)*Ri.transpose()).transpose()) - Re
2726 #
2721 #
2727 # #Check if there is a height between 70 and 110 km
2722 # #Check if there is a height between 70 and 110 km
2728 # h_bool = numpy.sum(numpy.logical_and(hi > minHeight, hi < maxHeight), axis = 1)
2723 # h_bool = numpy.sum(numpy.logical_and(hi > minHeight, hi < maxHeight), axis = 1)
2729 # ind_h = numpy.where(h_bool == 1)[0]
2724 # ind_h = numpy.where(h_bool == 1)[0]
2730 #
2725 #
2731 # hCorr = hi[ind_h, :]
2726 # hCorr = hi[ind_h, :]
2732 # ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight))
2727 # ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight))
2733 #
2728 #
2734 # hCorr = hi[ind_hCorr]
2729 # hCorr = hi[ind_hCorr]
2735 # heights[ind_h] = hCorr
2730 # heights[ind_h] = hCorr
2736 #
2731 #
2737 # #Setting Error
2732 # #Setting Error
2738 # #Number 13: Height unresolvable echo: not valid height within 70 to 110 km
2733 # #Number 13: Height unresolvable echo: not valid height within 70 to 110 km
2739 # #Number 14: Height ambiguous echo: more than one possible height within 70 to 110 km
2734 # #Number 14: Height ambiguous echo: more than one possible height within 70 to 110 km
2740 #
2735 #
2741 # indInvalid2 = numpy.where(numpy.logical_and(h_bool > 1, error == 0))[0]
2736 # indInvalid2 = numpy.where(numpy.logical_and(h_bool > 1, error == 0))[0]
2742 # error[indInvalid2] = 14
2737 # error[indInvalid2] = 14
2743 # indInvalid1 = numpy.where(numpy.logical_and(h_bool == 0, error == 0))[0]
2738 # indInvalid1 = numpy.where(numpy.logical_and(h_bool == 0, error == 0))[0]
2744 # error[indInvalid1] = 13
2739 # error[indInvalid1] = 13
2745 #
2740 #
2746 # return heights, error
2741 # return heights, error
Show file before | Show file after | Hide comments
@@ -1,393 +1,420
1 '''
1 '''
2 @author: Juan C. Espinoza
2 @author: Juan C. Espinoza
3 '''
3 '''
4
4
5 import time
5 import time
6 import json
6 import json
7 import numpy
7 import numpy
8 import paho.mqtt.client as mqtt
8 import paho.mqtt.client as mqtt
9 import zmq
9 import zmq
10 import cPickle as pickle
10 import cPickle as pickle
11 import datetime
11 import datetime
12 from zmq.utils.monitor import recv_monitor_message
12 from zmq.utils.monitor import recv_monitor_message
13 from functools import wraps
13 from functools import wraps
14 from threading import Thread
14 from threading import Thread
15 from multiprocessing import Process
15 from multiprocessing import Process
16
16
17 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
17 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
18
18
19 MAXNUMX = 100
19 MAXNUMX = 100
20 MAXNUMY = 100
20 MAXNUMY = 100
21
21
22 class PrettyFloat(float):
22 class PrettyFloat(float):
23 def __repr__(self):
23 def __repr__(self):
24 return '%.2f' % self
24 return '%.2f' % self
25
25
26 def roundFloats(obj):
26 def roundFloats(obj):
27 if isinstance(obj, list):
27 if isinstance(obj, list):
28 return map(roundFloats, obj)
28 return map(roundFloats, obj)
29 elif isinstance(obj, float):
29 elif isinstance(obj, float):
30 return round(obj, 2)
30 return round(obj, 2)
31
31
32 def decimate(z):
32 def decimate(z):
33 # dx = int(len(self.x)/self.__MAXNUMX) + 1
33 # dx = int(len(self.x)/self.__MAXNUMX) + 1
34 dy = int(len(z[0])/MAXNUMY) + 1
34 dy = int(len(z[0])/MAXNUMY) + 1
35 return z[::, ::dy]
35 return z[::, ::dy]
36
36
37 class throttle(object):
37 class throttle(object):
38 """Decorator that prevents a function from being called more than once every
38 """Decorator that prevents a function from being called more than once every
39 time period.
39 time period.
40 To create a function that cannot be called more than once a minute, but
40 To create a function that cannot be called more than once a minute, but
41 will sleep until it can be called:
41 will sleep until it can be called:
42 @throttle(minutes=1)
42 @throttle(minutes=1)
43 def foo():
43 def foo():
44 pass
44 pass
45
45
46 for i in range(10):
46 for i in range(10):
47 foo()
47 foo()
48 print "This function has run %s times." % i
48 print "This function has run %s times." % i
49 """
49 """
50
50
51 def __init__(self, seconds=0, minutes=0, hours=0):
51 def __init__(self, seconds=0, minutes=0, hours=0):
52 self.throttle_period = datetime.timedelta(
52 self.throttle_period = datetime.timedelta(
53 seconds=seconds, minutes=minutes, hours=hours
53 seconds=seconds, minutes=minutes, hours=hours
54 )
54 )
55
55
56 self.time_of_last_call = datetime.datetime.min
56 self.time_of_last_call = datetime.datetime.min
57
57
58 def __call__(self, fn):
58 def __call__(self, fn):
59 @wraps(fn)
59 @wraps(fn)
60 def wrapper(*args, **kwargs):
60 def wrapper(*args, **kwargs):
61 now = datetime.datetime.now()
61 now = datetime.datetime.now()
62 time_since_last_call = now - self.time_of_last_call
62 time_since_last_call = now - self.time_of_last_call
63 time_left = self.throttle_period - time_since_last_call
63 time_left = self.throttle_period - time_since_last_call
64
64
65 if time_left > datetime.timedelta(seconds=0):
65 if time_left > datetime.timedelta(seconds=0):
66 return
66 return
67
67
68 self.time_of_last_call = datetime.datetime.now()
68 self.time_of_last_call = datetime.datetime.now()
69 return fn(*args, **kwargs)
69 return fn(*args, **kwargs)
70
70
71 return wrapper
71 return wrapper
72
72
73
73
74 class PublishData(Operation):
74 class PublishData(Operation):
75 """Clase publish."""
75 """Clase publish."""
76
76
77 def __init__(self, **kwargs):
77 def __init__(self, **kwargs):
78 """Inicio."""
78 """Inicio."""
79 Operation.__init__(self, **kwargs)
79 Operation.__init__(self, **kwargs)
80 self.isConfig = False
80 self.isConfig = False
81 self.client = None
81 self.client = None
82 self.zeromq = None
82 self.zeromq = None
83 self.mqtt = None
83 self.mqtt = None
84
84
85 def on_disconnect(self, client, userdata, rc):
85 def on_disconnect(self, client, userdata, rc):
86 if rc != 0:
86 if rc != 0:
87 print("Unexpected disconnection.")
87 print("Unexpected disconnection.")
88 self.connect()
88 self.connect()
89
89
90 def connect(self):
90 def connect(self):
91 print 'trying to connect'
91 print 'trying to connect'
92 try:
92 try:
93 self.client.connect(
93 self.client.connect(
94 host=self.host,
94 host=self.host,
95 port=self.port,
95 port=self.port,
96 keepalive=60*10,
96 keepalive=60*10,
97 bind_address='')
97 bind_address='')
98 self.client.loop_start()
98 self.client.loop_start()
99 # self.client.publish(
99 # self.client.publish(
100 # self.topic + 'SETUP',
100 # self.topic + 'SETUP',
101 # json.dumps(setup),
101 # json.dumps(setup),
102 # retain=True
102 # retain=True
103 # )
103 # )
104 except:
104 except:
105 print "MQTT Conection error."
105 print "MQTT Conection error."
106 self.client = False
106 self.client = False
107
107
108 def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, **kwargs):
108 def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, **kwargs):
109 self.counter = 0
109 self.counter = 0
110 self.topic = kwargs.get('topic', 'schain')
110 self.topic = kwargs.get('topic', 'schain')
111 self.delay = kwargs.get('delay', 0)
111 self.delay = kwargs.get('delay', 0)
112 self.plottype = kwargs.get('plottype', 'spectra')
112 self.plottype = kwargs.get('plottype', 'spectra')
113 self.host = kwargs.get('host', "10.10.10.82")
113 self.host = kwargs.get('host', "10.10.10.82")
114 self.port = kwargs.get('port', 3000)
114 self.port = kwargs.get('port', 3000)
115 self.clientId = clientId
115 self.clientId = clientId
116 self.cnt = 0
116 self.cnt = 0
117 self.zeromq = zeromq
117 self.zeromq = zeromq
118 self.mqtt = kwargs.get('plottype', 0)
118 self.mqtt = kwargs.get('plottype', 0)
119 self.client = None
119 self.client = None
120 setup = []
120 setup = []
121 if mqtt is 1:
121 if mqtt is 1:
122 self.client = mqtt.Client(
122 self.client = mqtt.Client(
123 client_id=self.clientId + self.topic + 'SCHAIN',
123 client_id=self.clientId + self.topic + 'SCHAIN',
124 clean_session=True)
124 clean_session=True)
125 self.client.on_disconnect = self.on_disconnect
125 self.client.on_disconnect = self.on_disconnect
126 self.connect()
126 self.connect()
127 for plot in self.plottype:
127 for plot in self.plottype:
128 setup.append({
128 setup.append({
129 'plot': plot,
129 'plot': plot,
130 'topic': self.topic + plot,
130 'topic': self.topic + plot,
131 'title': getattr(self, plot + '_' + 'title', False),
131 'title': getattr(self, plot + '_' + 'title', False),
132 'xlabel': getattr(self, plot + '_' + 'xlabel', False),
132 'xlabel': getattr(self, plot + '_' + 'xlabel', False),
133 'ylabel': getattr(self, plot + '_' + 'ylabel', False),
133 'ylabel': getattr(self, plot + '_' + 'ylabel', False),
134 'xrange': getattr(self, plot + '_' + 'xrange', False),
134 'xrange': getattr(self, plot + '_' + 'xrange', False),
135 'yrange': getattr(self, plot + '_' + 'yrange', False),
135 'yrange': getattr(self, plot + '_' + 'yrange', False),
136 'zrange': getattr(self, plot + '_' + 'zrange', False),
136 'zrange': getattr(self, plot + '_' + 'zrange', False),
137 })
137 })
138 if zeromq is 1:
138 if zeromq is 1:
139 context = zmq.Context()
139 context = zmq.Context()
140 self.zmq_socket = context.socket(zmq.PUSH)
140 self.zmq_socket = context.socket(zmq.PUSH)
141 server = kwargs.get('server', 'zmq.pipe')
141 server = kwargs.get('server', 'zmq.pipe')
142
142
143 if 'tcp://' in server:
143 if 'tcp://' in server:
144 address = server
144 address = server
145 else:
145 else:
146 address = 'ipc:///tmp/%s' % server
146 address = 'ipc:///tmp/%s' % server
147
147
148 self.zmq_socket.connect(address)
148 self.zmq_socket.connect(address)
149 time.sleep(1)
149 time.sleep(1)
150
150
151
151
152
152
153 def publish_data(self):
153 def publish_data(self):
154 self.dataOut.finished = False
154 self.dataOut.finished = False
155 if self.mqtt is 1:
155 if self.mqtt is 1:
156 yData = self.dataOut.heightList[:2].tolist()
156 yData = self.dataOut.heightList[:2].tolist()
157 if self.plottype == 'spectra':
157 if self.plottype == 'spectra':
158 data = getattr(self.dataOut, 'data_spc')
158 data = getattr(self.dataOut, 'data_spc')
159 z = data/self.dataOut.normFactor
159 z = data/self.dataOut.normFactor
160 zdB = 10*numpy.log10(z)
160 zdB = 10*numpy.log10(z)
161 xlen, ylen = zdB[0].shape
161 xlen, ylen = zdB[0].shape
162 dx = int(xlen/MAXNUMX) + 1
162 dx = int(xlen/MAXNUMX) + 1
163 dy = int(ylen/MAXNUMY) + 1
163 dy = int(ylen/MAXNUMY) + 1
164 Z = [0 for i in self.dataOut.channelList]
164 Z = [0 for i in self.dataOut.channelList]
165 for i in self.dataOut.channelList:
165 for i in self.dataOut.channelList:
166 Z[i] = zdB[i][::dx, ::dy].tolist()
166 Z[i] = zdB[i][::dx, ::dy].tolist()
167 payload = {
167 payload = {
168 'timestamp': self.dataOut.utctime,
168 'timestamp': self.dataOut.utctime,
169 'data': roundFloats(Z),
169 'data': roundFloats(Z),
170 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
170 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
171 'interval': self.dataOut.getTimeInterval(),
171 'interval': self.dataOut.getTimeInterval(),
172 'type': self.plottype,
172 'type': self.plottype,
173 'yData': yData
173 'yData': yData
174 }
174 }
175 # print payload
175 # print payload
176
176
177 elif self.plottype in ('rti', 'power'):
177 elif self.plottype in ('rti', 'power'):
178 data = getattr(self.dataOut, 'data_spc')
178 data = getattr(self.dataOut, 'data_spc')
179 z = data/self.dataOut.normFactor
179 z = data/self.dataOut.normFactor
180 avg = numpy.average(z, axis=1)
180 avg = numpy.average(z, axis=1)
181 avgdB = 10*numpy.log10(avg)
181 avgdB = 10*numpy.log10(avg)
182 xlen, ylen = z[0].shape
182 xlen, ylen = z[0].shape
183 dy = numpy.floor(ylen/self.__MAXNUMY) + 1
183 dy = numpy.floor(ylen/self.__MAXNUMY) + 1
184 AVG = [0 for i in self.dataOut.channelList]
184 AVG = [0 for i in self.dataOut.channelList]
185 for i in self.dataOut.channelList:
185 for i in self.dataOut.channelList:
186 AVG[i] = avgdB[i][::dy].tolist()
186 AVG[i] = avgdB[i][::dy].tolist()
187 payload = {
187 payload = {
188 'timestamp': self.dataOut.utctime,
188 'timestamp': self.dataOut.utctime,
189 'data': roundFloats(AVG),
189 'data': roundFloats(AVG),
190 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
190 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
191 'interval': self.dataOut.getTimeInterval(),
191 'interval': self.dataOut.getTimeInterval(),
192 'type': self.plottype,
192 'type': self.plottype,
193 'yData': yData
193 'yData': yData
194 }
194 }
195 elif self.plottype == 'noise':
195 elif self.plottype == 'noise':
196 noise = self.dataOut.getNoise()/self.dataOut.normFactor
196 noise = self.dataOut.getNoise()/self.dataOut.normFactor
197 noisedB = 10*numpy.log10(noise)
197 noisedB = 10*numpy.log10(noise)
198 payload = {
198 payload = {
199 'timestamp': self.dataOut.utctime,
199 'timestamp': self.dataOut.utctime,
200 'data': roundFloats(noisedB.reshape(-1, 1).tolist()),
200 'data': roundFloats(noisedB.reshape(-1, 1).tolist()),
201 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
201 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
202 'interval': self.dataOut.getTimeInterval(),
202 'interval': self.dataOut.getTimeInterval(),
203 'type': self.plottype,
203 'type': self.plottype,
204 'yData': yData
204 'yData': yData
205 }
205 }
206 elif self.plottype == 'snr':
206 elif self.plottype == 'snr':
207 data = getattr(self.dataOut, 'data_SNR')
207 data = getattr(self.dataOut, 'data_SNR')
208 avgdB = 10*numpy.log10(data)
208 avgdB = 10*numpy.log10(data)
209
209
210 ylen = data[0].size
210 ylen = data[0].size
211 dy = numpy.floor(ylen/self.__MAXNUMY) + 1
211 dy = numpy.floor(ylen/self.__MAXNUMY) + 1
212 AVG = [0 for i in self.dataOut.channelList]
212 AVG = [0 for i in self.dataOut.channelList]
213 for i in self.dataOut.channelList:
213 for i in self.dataOut.channelList:
214 AVG[i] = avgdB[i][::dy].tolist()
214 AVG[i] = avgdB[i][::dy].tolist()
215 payload = {
215 payload = {
216 'timestamp': self.dataOut.utctime,
216 'timestamp': self.dataOut.utctime,
217 'data': roundFloats(AVG),
217 'data': roundFloats(AVG),
218 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
218 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
219 'type': self.plottype,
219 'type': self.plottype,
220 'yData': yData
220 'yData': yData
221 }
221 }
222 else:
222 else:
223 print "Tipo de grafico invalido"
223 print "Tipo de grafico invalido"
224 payload = {
224 payload = {
225 'data': 'None',
225 'data': 'None',
226 'timestamp': 'None',
226 'timestamp': 'None',
227 'type': None
227 'type': None
228 }
228 }
229 # print 'Publishing data to {}'.format(self.host)
229 # print 'Publishing data to {}'.format(self.host)
230 self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0)
230 self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0)
231
231
232 if self.zeromq is 1:
232 if self.zeromq is 1:
233 print '[Sending] {} - {}'.format(self.dataOut.type, self.dataOut.datatime)
233 print '[Sending] {} - {}'.format(self.dataOut.type, self.dataOut.datatime)
234 self.zmq_socket.send_pyobj(self.dataOut)
234 self.zmq_socket.send_pyobj(self.dataOut)
235
235
236 def run(self, dataOut, **kwargs):
236 def run(self, dataOut, **kwargs):
237 self.dataOut = dataOut
237 self.dataOut = dataOut
238 if not self.isConfig:
238 if not self.isConfig:
239 self.setup(**kwargs)
239 self.setup(**kwargs)
240 self.isConfig = True
240 self.isConfig = True
241
241
242 self.publish_data()
242 self.publish_data()
243 time.sleep(self.delay)
243 time.sleep(self.delay)
244
244
245 def close(self):
245 def close(self):
246 if self.zeromq is 1:
246 if self.zeromq is 1:
247 self.dataOut.finished = True
247 self.dataOut.finished = True
248 self.zmq_socket.send_pyobj(self.dataOut)
248 self.zmq_socket.send_pyobj(self.dataOut)
249
249
250 if self.client:
250 if self.client:
251 self.client.loop_stop()
251 self.client.loop_stop()
252 self.client.disconnect()
252 self.client.disconnect()
253
253
254
254
255 class ReceiverData(ProcessingUnit, Process):
255 class ReceiverData(ProcessingUnit, Process):
256
256
257 throttle_value = 5
257 throttle_value = 5
258
258
259 def __init__(self, **kwargs):
259 def __init__(self, **kwargs):
260
260
261 ProcessingUnit.__init__(self, **kwargs)
261 ProcessingUnit.__init__(self, **kwargs)
262 Process.__init__(self)
262 Process.__init__(self)
263 self.mp = False
263 self.mp = False
264 self.isConfig = False
264 self.isConfig = False
265 self.isWebConfig = False
265 self.plottypes =[]
266 self.plottypes =[]
266 self.connections = 0
267 self.connections = 0
267 server = kwargs.get('server', 'zmq.pipe')
268 server = kwargs.get('server', 'zmq.pipe')
269 plot_server = kwargs.get('plot_server', 'zmq.web')
268 if 'tcp://' in server:
270 if 'tcp://' in server:
269 address = server
271 address = server
270 else:
272 else:
271 address = 'ipc:///tmp/%s' % server
273 address = 'ipc:///tmp/%s' % server
272
274
275 if 'tcp://' in plot_server:
276 plot_address = plot_server
277 else:
278 plot_address = 'ipc:///tmp/%s' % plot_server
279
273 self.address = address
280 self.address = address
281 self.plot_address = plot_address
274 self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')]
282 self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')]
275 self.realtime = kwargs.get('realtime', False)
283 self.realtime = kwargs.get('realtime', False)
276 self.throttle_value = kwargs.get('throttle', 10)
284 self.throttle_value = kwargs.get('throttle', 10)
277 self.sendData = self.initThrottle(self.throttle_value)
285 self.sendData = self.initThrottle(self.throttle_value)
278 self.setup()
286 self.setup()
279
287
280 def setup(self):
288 def setup(self):
281
289
282 self.data = {}
290 self.data = {}
283 self.data['times'] = []
291 self.data['times'] = []
284 for plottype in self.plottypes:
292 for plottype in self.plottypes:
285 self.data[plottype] = {}
293 self.data[plottype] = {}
286 self.data['noise'] = {}
294 self.data['noise'] = {}
287 self.data['throttle'] = self.throttle_value
295 self.data['throttle'] = self.throttle_value
288 self.data['ENDED'] = False
296 self.data['ENDED'] = False
289 self.isConfig = True
297 self.isConfig = True
290 self.data_web = {}
298 self.data_web = {}
291
299
292 def event_monitor(self, monitor):
300 def event_monitor(self, monitor):
293
301
294 events = {}
302 events = {}
295
303
296 for name in dir(zmq):
304 for name in dir(zmq):
297 if name.startswith('EVENT_'):
305 if name.startswith('EVENT_'):
298 value = getattr(zmq, name)
306 value = getattr(zmq, name)
299 events[value] = name
307 events[value] = name
300
308
301 while monitor.poll():
309 while monitor.poll():
302 evt = recv_monitor_message(monitor)
310 evt = recv_monitor_message(monitor)
303 if evt['event'] == 32:
311 if evt['event'] == 32:
304 self.connections += 1
312 self.connections += 1
305 if evt['event'] == 512:
313 if evt['event'] == 512:
306 pass
314 pass
307 if self.connections == 0 and self.started is True:
315 if self.connections == 0 and self.started is True:
308 self.ended = True
316 self.ended = True
309 # send('ENDED')
317 # send('ENDED')
310 evt.update({'description': events[evt['event']]})
318 evt.update({'description': events[evt['event']]})
311
319
312 if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
320 if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
313 break
321 break
314 monitor.close()
322 monitor.close()
315 print("event monitor thread done!")
323 print("event monitor thread done!")
316
324
317 def initThrottle(self, throttle_value):
325 def initThrottle(self, throttle_value):
318
326
319 @throttle(seconds=throttle_value)
327 @throttle(seconds=throttle_value)
320 def sendDataThrottled(fn_sender, data):
328 def sendDataThrottled(fn_sender, data):
321 fn_sender(data)
329 fn_sender(data)
322
330
323 return sendDataThrottled
331 return sendDataThrottled
324
332
325 def send(self, data):
333 def send(self, data):
326 # print '[sending] data=%s size=%s' % (data.keys(), len(data['times']))
334 # print '[sending] data=%s size=%s' % (data.keys(), len(data['times']))
327 self.sender.send_pyobj(data)
335 self.sender.send_pyobj(data)
328
336
329 def update(self):
337 def update(self):
330
338
331 t = self.dataOut.ltctime
339 t = self.dataOut.ltctime
332 self.data['times'].append(t)
340 self.data['times'].append(t)
333 self.data['dataOut'] = self.dataOut
341 self.data['dataOut'] = self.dataOut
334 for plottype in self.plottypes:
342 for plottype in self.plottypes:
335 if plottype == 'spc':
343 if plottype == 'spc':
336 z = self.dataOut.data_spc/self.dataOut.normFactor
344 z = self.dataOut.data_spc/self.dataOut.normFactor
337 self.data[plottype] = 10*numpy.log10(z)
345 self.data[plottype] = 10*numpy.log10(z)
338 self.data['noise'][t] = 10*numpy.log10(self.dataOut.getNoise()/self.dataOut.normFactor)
346 self.data['noise'][t] = 10*numpy.log10(self.dataOut.getNoise()/self.dataOut.normFactor)
339 if plottype == 'rti':
347 if plottype == 'rti':
340 self.data[plottype][t] = self.dataOut.getPower()
348 self.data[plottype][t] = self.dataOut.getPower()
341 if plottype == 'snr':
349 if plottype == 'snr':
342 self.data[plottype][t] = 10*numpy.log10(self.dataOut.data_SNR)
350 self.data[plottype][t] = 10*numpy.log10(self.dataOut.data_SNR)
343 if plottype == 'dop':
351 if plottype == 'dop':
344 self.data[plottype][t] = 10*numpy.log10(self.dataOut.data_DOP)
352 self.data[plottype][t] = 10*numpy.log10(self.dataOut.data_DOP)
345 if plottype == 'coh':
353 if plottype == 'coh':
346 self.data[plottype][t] = self.dataOut.getCoherence()
354 self.data[plottype][t] = self.dataOut.getCoherence()
347 if plottype == 'phase':
355 if plottype == 'phase':
348 self.data[plottype][t] = self.dataOut.getCoherence(phase=True)
356 self.data[plottype][t] = self.dataOut.getCoherence(phase=True)
349 if self.realtime:
357 if self.realtime:
350 self.data_web[plottype] = roundFloats(decimate(self.data[plottype][t]).tolist())
358 self.data_web[plottype] = roundFloats(decimate(self.data[plottype][t]).tolist())
351 self.data_web['time'] = t
359 self.data_web['time'] = t
352 def run(self):
360 def run(self):
353
361
354 print '[Starting] {} from {}'.format(self.name, self.address)
362 print '[Starting] {} from {}'.format(self.name, self.address)
355
363
356 self.context = zmq.Context()
364 self.context = zmq.Context()
357 self.receiver = self.context.socket(zmq.PULL)
365 self.receiver = self.context.socket(zmq.PULL)
358 self.receiver.bind(self.address)
366 self.receiver.bind(self.address)
359 monitor = self.receiver.get_monitor_socket()
367 monitor = self.receiver.get_monitor_socket()
360 self.sender = self.context.socket(zmq.PUB)
368 self.sender = self.context.socket(zmq.PUB)
361 if self.realtime:
369 if self.realtime:
362 self.sender_web = self.context.socket(zmq.PUB)
370 self.sender_web = self.context.socket(zmq.PUB)
363 # self.sender_web.setsockopt(zmq.PUBLISH, 'realtime')
371 self.sender_web.bind(self.plot_address)
364 self.sender_web.bind("ipc:///tmp/zmq.web")
365 self.sender.bind("ipc:///tmp/zmq.plots")
372 self.sender.bind("ipc:///tmp/zmq.plots")
366
373
367 t = Thread(target=self.event_monitor, args=(monitor,))
374 t = Thread(target=self.event_monitor, args=(monitor,))
368 t.start()
375 t.start()
369
376
370 while True:
377 while True:
371 self.dataOut = self.receiver.recv_pyobj()
378 self.dataOut = self.receiver.recv_pyobj()
372 # print '[Receiving] {} - {}'.format(self.dataOut.type,
379 # print '[Receiving] {} - {}'.format(self.dataOut.type,
373 # self.dataOut.datatime.ctime())
380 # self.dataOut.datatime.ctime())
374
381
375 self.update()
382 self.update()
376
383
377 if self.dataOut.finished is True:
384 if self.dataOut.finished is True:
378 self.send(self.data)
385 self.send(self.data)
379 self.connections -= 1
386 self.connections -= 1
380 if self.connections == 0 and self.started:
387 if self.connections == 0 and self.started:
381 self.ended = True
388 self.ended = True
382 self.data['ENDED'] = True
389 self.data['ENDED'] = True
383 self.send(self.data)
390 self.send(self.data)
384 self.setup()
391 self.setup()
385 else:
392 else:
386 if self.realtime:
393 if self.realtime:
387 self.send(self.data)
394 self.send(self.data)
388 self.sender_web.send_string(json.dumps(self.data_web))
395 self.sender_web.send_string(json.dumps(self.data_web))
389 else:
396 else:
390 self.sendData(self.send, self.data)
397 self.sendData(self.send, self.data)
391 self.started = True
398 self.started = True
392
399
393 return
400 return
401
402 def sendToWeb(self):
403
404 if not self.isWebConfig:
405 context = zmq.Context()
406 sender_web_config = context.socket(zmq.PUB)
407 if 'tcp://' in self.plot_address:
408 print self.plot_address
409 dum, address, port = self.plot_address.split(':')
410 conf_address = '{}:{}:{}'.format(dum, address, int(port)+1)
411 else:
412 conf_address = self.plot_address + '.config'
413 sender_web_config.bind(conf_address)
414
415 for kwargs in self.operationKwargs.values():
416 if 'plot' in kwargs:
417 sender_web_config.send_string(json.dumps(kwargs))
418 print kwargs
419 self.isWebConfig = True
420
General Comments 0
You need to be logged in to leave comments. Login now